4 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 * This program is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 only,
8 * as published by the Free Software Foundation.
10 * This program is distributed in the hope that it will be useful, but
11 * WITHOUT ANY WARRANTY; without even the implied warranty of
12 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU
13 * General Public License version 2 for more details (a copy is included
14 * in the LICENSE file that accompanied this code).
16 * You should have received a copy of the GNU General Public License
17 * version 2 along with this program; if not, write to the
18 * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
19 * Boston, MA 021110-1307, USA
24 * Copyright (c) 2012, 2017, Intel Corporation.
25 * Use is subject to license terms.
27 * Author: Johann Lombardi <johann.lombardi@intel.com>
28 * Author: Niu Yawei <yawei.niu@intel.com>
31 #define DEBUG_SUBSYSTEM S_LQUOTA
33 #include <obd_class.h>
34 #include "qmt_internal.h"
37 * Retrieve quota settings for a given identifier.
39 * \param env - is the environment passed by the caller
40 * \param qmt - is the quota master target
41 * \param pool_id - is the 16-bit pool identifier
42 * \param restype - is the pool type, either block (i.e. LQUOTA_RES_DT) or inode
43 * (i.e. LQUOTA_RES_MD)
44 * \param qtype - is the quota type
45 * \param id - is the quota indentifier for which we want to acces quota
47 * \param hard - is the output variable where to copy the hard limit
48 * \param soft - is the output variable where to copy the soft limit
49 * \param time - is the output variable where to copy the grace time
51 static int qmt_get(const struct lu_env *env, struct qmt_device *qmt,
52 __u16 pool_id, __u8 restype, __u8 qtype, union lquota_id *id,
53 __u64 *hard, __u64 *soft, __u64 *time)
55 struct lquota_entry *lqe;
58 /* look-up lqe structure containing quota settings */
59 lqe = qmt_pool_lqe_lookup(env, qmt, pool_id, restype, qtype, id);
63 /* copy quota settings */
65 LQUOTA_DEBUG(lqe, "fetch settings");
67 *hard = lqe->lqe_hardlimit;
69 *soft = lqe->lqe_softlimit;
71 *time = lqe->lqe_gracetime;
79 * Update quota settings for a given identifier.
81 * \param env - is the environment passed by the caller
82 * \param qmt - is the quota master target
83 * \param pool_id - is the 16-bit pool identifier
84 * \param restype - is the pool type, either block (i.e. LQUOTA_RES_DT) or inode
85 * (i.e. LQUOTA_RES_MD)
86 * \param qtype - is the quota type
87 * \param id - is the quota indentifier for which we want to modify quota
89 * \param hard - is the new hard limit
90 * \param soft - is the new soft limit
91 * \param time - is the new grace time
92 * \param valid - is the list of settings to change
94 static int qmt_set(const struct lu_env *env, struct qmt_device *qmt,
95 __u16 pool_id, __u8 restype, __u8 qtype,
96 union lquota_id *id, __u64 hard, __u64 soft, __u64 time,
99 struct qmt_thread_info *qti = qmt_info(env);
100 struct lquota_entry *lqe;
101 struct thandle *th = NULL;
104 bool dirtied = false;
108 /* look-up quota entry associated with this ID */
109 lqe = qmt_pool_lqe_lookup(env, qmt, pool_id, restype, qtype, id);
111 RETURN(PTR_ERR(lqe));
113 /* allocate & start transaction with enough credits to update quota
114 * settings in the global index file */
115 th = qmt_trans_start(env, lqe, &qti->qti_restore);
117 GOTO(out_nolock, rc = PTR_ERR(th));
119 now = ktime_get_real_seconds();
122 LQUOTA_DEBUG(lqe, "changing quota settings valid:%x hard:%llu soft:"
123 "%llu time:%llu", valid, hard, soft, time);
125 if ((valid & QIF_TIMES) != 0 && lqe->lqe_gracetime != time) {
126 /* change time settings */
127 lqe->lqe_gracetime = time;
131 if ((valid & QIF_LIMITS) != 0 &&
132 (lqe->lqe_hardlimit != hard || lqe->lqe_softlimit != soft)) {
133 rc = qmt_validate_limits(lqe, hard, soft);
137 /* recompute qunit in case it was never initialized */
138 qmt_revalidate(env, lqe);
140 /* change quota limits */
141 lqe->lqe_hardlimit = hard;
142 lqe->lqe_softlimit = soft;
144 /* clear grace time */
145 if (lqe->lqe_softlimit == 0 ||
146 lqe->lqe_granted <= lqe->lqe_softlimit)
147 /* no soft limit or below soft limit, let's clear grace
149 lqe->lqe_gracetime = 0;
150 else if ((valid & QIF_TIMES) == 0)
151 /* set grace only if user hasn't provided his own */
152 lqe->lqe_gracetime = now + qmt_lqe_grace(lqe);
154 /* change enforced status based on new parameters */
155 if (lqe->lqe_hardlimit == 0 && lqe->lqe_softlimit == 0)
156 lqe->lqe_enforced = false;
158 lqe->lqe_enforced = true;
164 /* write new quota settings to disk */
165 rc = qmt_glb_write(env, th, lqe, LQUOTA_BUMP_VER, &ver);
167 /* restore initial quota settings */
168 qmt_restore(lqe, &qti->qti_restore);
172 /* compute new qunit value now that we have modified the quota
174 qmt_adjust_qunit(env, lqe);
176 /* clear/set edquot flag as needed */
177 qmt_adjust_edquot(lqe, now);
181 lqe_write_unlock(lqe);
185 if (th != NULL && !IS_ERR(th))
186 dt_trans_stop(env, qmt->qmt_child, th);
188 if (rc == 0 && dirtied)
189 qmt_glb_lock_notify(env, lqe, ver);
195 * Handle quotactl request.
197 * \param env - is the environment passed by the caller
198 * \param ld - is the lu device associated with the qmt
199 * \param oqctl - is the quotactl request
201 static int qmt_quotactl(const struct lu_env *env, struct lu_device *ld,
202 struct obd_quotactl *oqctl)
204 struct qmt_thread_info *qti = qmt_info(env);
205 union lquota_id *id = &qti->qti_id;
206 struct qmt_device *qmt = lu2qmt_dev(ld);
207 struct obd_dqblk *dqb = &oqctl->qc_dqblk;
211 LASSERT(qmt != NULL);
213 if (oqctl->qc_type >= LL_MAXQUOTAS)
214 /* invalid quota type */
217 switch (oqctl->qc_cmd) {
219 case Q_GETINFO: /* read grace times */
220 /* Global grace time is stored in quota settings of ID 0. */
223 /* read inode grace time */
224 rc = qmt_get(env, qmt, 0, LQUOTA_RES_MD, oqctl->qc_type, id,
225 NULL, NULL, &oqctl->qc_dqinfo.dqi_igrace);
229 /* read block grace time */
230 rc = qmt_get(env, qmt, 0, LQUOTA_RES_DT, oqctl->qc_type, id,
231 NULL, NULL, &oqctl->qc_dqinfo.dqi_bgrace);
234 case Q_SETINFO: /* modify grace times */
235 /* setinfo should be using dqi->dqi_valid, but lfs incorrectly
236 * sets the valid flags in dqb->dqb_valid instead, try to live
239 /* Global grace time is stored in quota settings of ID 0. */
242 if ((dqb->dqb_valid & QIF_ITIME) != 0) {
243 /* set inode grace time */
244 rc = qmt_set(env, qmt, 0, LQUOTA_RES_MD, oqctl->qc_type,
245 id, 0, 0, oqctl->qc_dqinfo.dqi_igrace,
251 if ((dqb->dqb_valid & QIF_BTIME) != 0)
252 /* set block grace time */
253 rc = qmt_set(env, qmt, 0, LQUOTA_RES_DT, oqctl->qc_type,
254 id, 0, 0, oqctl->qc_dqinfo.dqi_bgrace,
258 case Q_GETQUOTA: /* consult quota limit */
259 /* There is no quota limit for root user & group */
260 if (oqctl->qc_id == 0) {
261 memset(dqb, 0, sizeof(*dqb));
262 dqb->dqb_valid = QIF_LIMITS | QIF_TIMES;
265 /* extract quota ID from quotactl request */
266 id->qid_uid = oqctl->qc_id;
268 /* look-up inode quota settings */
269 rc = qmt_get(env, qmt, 0, LQUOTA_RES_MD, oqctl->qc_type, id,
270 &dqb->dqb_ihardlimit, &dqb->dqb_isoftlimit,
275 dqb->dqb_valid |= QIF_ILIMITS | QIF_ITIME;
276 /* master isn't aware of actual inode usage */
277 dqb->dqb_curinodes = 0;
279 /* look-up block quota settings */
280 rc = qmt_get(env, qmt, 0, LQUOTA_RES_DT, oqctl->qc_type, id,
281 &dqb->dqb_bhardlimit, &dqb->dqb_bsoftlimit,
286 dqb->dqb_valid |= QIF_BLIMITS | QIF_BTIME;
287 /* master doesn't know the actual block usage */
288 dqb->dqb_curspace = 0;
291 case Q_SETQUOTA: /* change quota limits */
292 if (oqctl->qc_id == 0)
293 /* can't enforce a quota limit for root user & group */
295 /* extract quota ID from quotactl request */
296 id->qid_uid = oqctl->qc_id;
298 if ((dqb->dqb_valid & QIF_IFLAGS) != 0) {
299 /* update inode quota settings */
300 rc = qmt_set(env, qmt, 0, LQUOTA_RES_MD, oqctl->qc_type,
301 id, dqb->dqb_ihardlimit,
302 dqb->dqb_isoftlimit, dqb->dqb_itime,
303 dqb->dqb_valid & QIF_IFLAGS);
308 if ((dqb->dqb_valid & QIF_BFLAGS) != 0)
309 /* update block quota settings */
310 rc = qmt_set(env, qmt, 0, LQUOTA_RES_DT, oqctl->qc_type,
311 id, dqb->dqb_bhardlimit,
312 dqb->dqb_bsoftlimit, dqb->dqb_btime,
313 dqb->dqb_valid & QIF_BFLAGS);
317 CERROR("%s: unsupported quotactl command: %d\n",
318 qmt->qmt_svname, oqctl->qc_cmd);
326 * Helper function to handle quota request from slave.
328 * \param env - is the environment passed by the caller
329 * \param lqe - is the lquota_entry subject to the quota request
330 * \param qmt - is the master device
331 * \param uuid - is the uuid associated with the slave
332 * \param qb_flags - are the quota request flags as packed in the quota_body
333 * \param qb_count - is the amount of quota space the slave wants to
335 * \param qb_usage - is the current space usage on the slave
336 * \param repbody - is the quota_body of reply
338 * \retval 0 : success
339 * \retval -EDQUOT : out of quota
340 * -EINPROGRESS : inform client to retry write/create
341 * -ve : other appropriate errors
343 int qmt_dqacq0(const struct lu_env *env, struct lquota_entry *lqe,
344 struct qmt_device *qmt, struct obd_uuid *uuid, __u32 qb_flags,
345 __u64 qb_count, __u64 qb_usage, struct quota_body *repbody)
347 struct qmt_thread_info *qti = qmt_info(env);
349 struct dt_object *slv_obj = NULL;
350 __u64 slv_granted, slv_granted_bck;
351 struct thandle *th = NULL;
355 LASSERT(uuid != NULL);
357 /* initialize reply */
358 memset(repbody, 0, sizeof(*repbody));
359 memcpy(&repbody->qb_id, &lqe->lqe_id, sizeof(repbody->qb_id));
361 if (OBD_FAIL_CHECK(OBD_FAIL_QUOTA_RECOVERABLE_ERR))
362 RETURN(-cfs_fail_val);
364 /* look-up index file associated with acquiring slave */
365 slv_obj = lquota_disk_slv_find(env, qmt->qmt_child, LQE_ROOT(lqe),
366 lu_object_fid(&LQE_GLB_OBJ(lqe)->do_lu),
369 GOTO(out, rc = PTR_ERR(slv_obj));
371 /* pack slave fid in reply just for sanity check */
372 memcpy(&repbody->qb_slv_fid, lu_object_fid(&slv_obj->do_lu),
373 sizeof(struct lu_fid));
375 /* allocate & start transaction with enough credits to update
376 * global & slave indexes */
377 th = qmt_trans_start_with_slv(env, lqe, slv_obj, &qti->qti_restore);
379 GOTO(out, rc = PTR_ERR(th));
382 LQUOTA_DEBUG(lqe, "dqacq starts uuid:%s flags:0x%x wanted:%llu"
383 " usage:%llu", obd_uuid2str(uuid), qb_flags, qb_count,
386 /* Legal race, limits have been removed on master, but slave didn't
387 * receive the change yet. Just return EINPROGRESS until the slave gets
389 if (!lqe->lqe_enforced && !req_is_rel(qb_flags))
390 GOTO(out_locked, rc = -ESRCH);
392 /* recompute qunit in case it was never initialized */
393 qmt_revalidate(env, lqe);
395 /* slave just wants to acquire per-ID lock */
396 if (req_is_acq(qb_flags) && qb_count == 0)
397 GOTO(out_locked, rc = 0);
399 /* fetch how much quota space is already granted to this slave */
400 rc = qmt_slv_read(env, lqe, slv_obj, &slv_granted);
402 LQUOTA_ERROR(lqe, "Failed to get granted for slave %s, rc=%d",
403 obd_uuid2str(uuid), rc);
404 GOTO(out_locked, rc);
406 /* recall how much space this slave currently owns in order to restore
407 * it in case of failure */
408 slv_granted_bck = slv_granted;
410 /* record current time for soft limit & grace time management */
411 now = ktime_get_real_seconds();
413 if (req_is_rel(qb_flags)) {
414 /* Slave would like to release quota space */
415 if (slv_granted < qb_count ||
416 lqe->lqe_granted < qb_count) {
417 /* can't release more than granted */
418 LQUOTA_ERROR(lqe, "Release too much! uuid:%s release:"
419 "%llu granted:%llu, total:%llu",
420 obd_uuid2str(uuid), qb_count,
421 slv_granted, lqe->lqe_granted);
422 GOTO(out_locked, rc = -EINVAL);
425 repbody->qb_count = qb_count;
426 /* put released space back to global pool */
427 QMT_REL(lqe, slv_granted, qb_count);
428 GOTO(out_write, rc = 0);
431 if (req_has_rep(qb_flags) && slv_granted < qb_usage) {
432 /* Slave is reporting space usage in quota request and it turns
433 * out to be using more quota space than owned, so we adjust
434 * granted space regardless of the current state of affairs */
435 repbody->qb_count = qb_usage - slv_granted;
436 QMT_GRANT(lqe, slv_granted, repbody->qb_count);
439 if (!req_is_acq(qb_flags) && !req_is_preacq(qb_flags))
440 GOTO(out_write, rc = 0);
442 qmt_adjust_edquot(lqe, now);
444 /* no hope to claim further space back */
445 GOTO(out_write, rc = -EDQUOT);
447 if (qmt_space_exhausted(lqe, now)) {
448 /* might have some free space once rebalancing is completed */
449 rc = req_is_acq(qb_flags) ? -EINPROGRESS : -EDQUOT;
453 if (req_is_preacq(qb_flags)) {
454 /* slave would like to pre-acquire quota space. To do so, it
455 * reports in qb_count how much spare quota space it owns and we
456 * can grant back quota space which is consistent with qunit
459 if (qb_count >= lqe->lqe_qunit)
460 /* slave already own the maximum it should */
461 GOTO(out_write, rc = 0);
463 count = qmt_alloc_expand(lqe, slv_granted, qb_count);
465 GOTO(out_write, rc = -EDQUOT);
467 repbody->qb_count += count;
468 QMT_GRANT(lqe, slv_granted, count);
469 GOTO(out_write, rc = 0);
472 /* processing acquire request with clients waiting */
473 if (lqe->lqe_hardlimit != 0 &&
474 lqe->lqe_granted + qb_count > lqe->lqe_hardlimit) {
475 /* cannot grant as much as asked, but can still afford to grant
476 * some quota space back */
477 count = lqe->lqe_hardlimit - lqe->lqe_granted;
478 repbody->qb_count += count;
479 QMT_GRANT(lqe, slv_granted, count);
480 GOTO(out_write, rc = 0);
483 /* Whouhou! we can satisfy the slave request! */
484 repbody->qb_count += qb_count;
485 QMT_GRANT(lqe, slv_granted, qb_count);
487 /* Try to expand the acquired count for DQACQ */
488 count = qmt_alloc_expand(lqe, slv_granted, 0);
490 /* can even grant more than asked, it is like xmas ... */
491 repbody->qb_count += count;
492 QMT_GRANT(lqe, slv_granted, count);
493 GOTO(out_write, rc = 0);
496 GOTO(out_write, rc = 0);
498 if (repbody->qb_count == 0)
499 GOTO(out_locked, rc);
501 /* start/stop grace timer if required */
502 if (lqe->lqe_softlimit != 0) {
503 if (lqe->lqe_granted > lqe->lqe_softlimit &&
504 lqe->lqe_gracetime == 0)
505 /* first time over soft limit, let's start grace
507 lqe->lqe_gracetime = now + qmt_lqe_grace(lqe);
508 else if (lqe->lqe_granted <= lqe->lqe_softlimit &&
509 lqe->lqe_gracetime != 0)
510 /* Clear grace timer */
511 lqe->lqe_gracetime = 0;
514 /* Update slave index first since it is easier to roll back */
515 ret = qmt_slv_write(env, th, lqe, slv_obj, LQUOTA_BUMP_VER,
516 &repbody->qb_slv_ver, slv_granted);
518 /* restore initial quota settings */
519 qmt_restore(lqe, &qti->qti_restore);
521 repbody->qb_count = 0;
522 GOTO(out_locked, rc = ret);
525 /* Update global index, no version bump needed */
526 ret = qmt_glb_write(env, th, lqe, 0, NULL);
529 /* restore initial quota settings */
530 qmt_restore(lqe, &qti->qti_restore);
532 repbody->qb_count = 0;
534 /* restore previous granted value */
535 ret = qmt_slv_write(env, th, lqe, slv_obj, 0, NULL,
538 LQUOTA_ERROR(lqe, "failed to restore initial slave "
539 "value rc:%d ret%d", rc, ret);
542 qmt_adjust_edquot(lqe, now);
543 GOTO(out_locked, rc);
546 /* Total granted has been changed, let's try to adjust the qunit
547 * size according to the total granted & limits. */
548 qmt_adjust_qunit(env, lqe);
550 /* clear/set edquot flag and notify slaves via glimpse if needed */
551 qmt_adjust_edquot(lqe, now);
553 LQUOTA_DEBUG(lqe, "dqacq ends count:%llu ver:%llu rc:%d",
554 repbody->qb_count, repbody->qb_slv_ver, rc);
555 lqe_write_unlock(lqe);
557 if (th != NULL && !IS_ERR(th))
558 dt_trans_stop(env, qmt->qmt_child, th);
560 if (slv_obj != NULL && !IS_ERR(slv_obj))
561 dt_object_put(env, slv_obj);
563 if ((req_is_acq(qb_flags) || req_is_preacq(qb_flags)) &&
564 OBD_FAIL_CHECK(OBD_FAIL_QUOTA_EDQUOT)) {
565 /* introduce inconsistency between granted value in slave index
566 * and slave index copy of slave */
567 repbody->qb_count = 0;
575 * Handle quota request from slave.
577 * \param env - is the environment passed by the caller
578 * \param ld - is the lu device associated with the qmt
579 * \param req - is the quota acquire request
581 static int qmt_dqacq(const struct lu_env *env, struct lu_device *ld,
582 struct ptlrpc_request *req)
584 struct qmt_device *qmt = lu2qmt_dev(ld);
585 struct quota_body *qbody, *repbody;
586 struct obd_uuid *uuid;
587 struct ldlm_lock *lock;
588 struct lquota_entry *lqe;
589 int pool_id, pool_type, qtype;
593 qbody = req_capsule_client_get(&req->rq_pill, &RMF_QUOTA_BODY);
595 RETURN(err_serious(-EPROTO));
597 repbody = req_capsule_server_get(&req->rq_pill, &RMF_QUOTA_BODY);
599 RETURN(err_serious(-EFAULT));
601 /* verify if global lock is stale */
602 if (!lustre_handle_is_used(&qbody->qb_glb_lockh))
605 lock = ldlm_handle2lock(&qbody->qb_glb_lockh);
610 uuid = &req->rq_export->exp_client_uuid;
612 if (req_is_rel(qbody->qb_flags) + req_is_acq(qbody->qb_flags) +
613 req_is_preacq(qbody->qb_flags) > 1) {
614 CERROR("%s: malformed quota request with conflicting flags set "
615 "(%x) from slave %s\n", qmt->qmt_svname,
616 qbody->qb_flags, obd_uuid2str(uuid));
620 if (req_is_acq(qbody->qb_flags) || req_is_preacq(qbody->qb_flags)) {
621 /* acquire and pre-acquire should use a valid ID lock */
623 if (!lustre_handle_is_used(&qbody->qb_lockh))
626 lock = ldlm_handle2lock(&qbody->qb_lockh);
628 /* no lock associated with this handle */
631 LDLM_DEBUG(lock, "%sacquire request",
632 req_is_preacq(qbody->qb_flags) ? "pre" : "");
634 if (!obd_uuid_equals(&lock->l_export->exp_client_uuid, uuid)) {
635 /* sorry, no way to cheat ... */
640 if (ldlm_is_ast_sent(lock)) {
641 struct ptlrpc_service_part *svc;
644 svc = req->rq_rqbd->rqbd_svcpt;
645 timeout = at_est2timeout(at_get(&svc->scp_at_estimate));
646 timeout += (ldlm_bl_timeout(lock) >> 1);
648 /* lock is being cancelled, prolong timeout */
649 ldlm_refresh_waiting_lock(lock, timeout);
654 /* extract pool & quota information from global index FID packed in the
656 rc = lquota_extract_fid(&qbody->qb_fid, &pool_id, &pool_type, &qtype);
660 /* Find the quota entry associated with the quota id */
661 lqe = qmt_pool_lqe_lookup(env, qmt, pool_id, pool_type, qtype,
664 RETURN(PTR_ERR(lqe));
666 /* process quota request */
667 rc = qmt_dqacq0(env, lqe, qmt, uuid, qbody->qb_flags, qbody->qb_count,
668 qbody->qb_usage, repbody);
670 if (lustre_handle_is_used(&qbody->qb_lockh))
671 /* return current qunit value only to slaves owning an per-ID
672 * quota lock. For enqueue, the qunit value will be returned in
674 repbody->qb_qunit = lqe->lqe_qunit;
679 /* Vector of quota request handlers. This vector is used by the MDT to forward
680 * requests to the quota master. */
681 struct qmt_handlers qmt_hdls = {
682 /* quota request handlers */
683 .qmth_quotactl = qmt_quotactl,
684 .qmth_dqacq = qmt_dqacq,
687 .qmth_intent_policy = qmt_intent_policy,
688 .qmth_lvbo_init = qmt_lvbo_init,
689 .qmth_lvbo_update = qmt_lvbo_update,
690 .qmth_lvbo_size = qmt_lvbo_size,
691 .qmth_lvbo_fill = qmt_lvbo_fill,
692 .qmth_lvbo_free = qmt_lvbo_free,
694 EXPORT_SYMBOL(qmt_hdls);