lustre/quota/quota_interface.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  30  * Use is subject to license terms.
  31  *
  32  * Copyright (c) 2011, 2012, Whamcloud, Inc.
  33  */
  34 /*
  35  * This file is part of Lustre, http://www.lustre.org/
  36  * Lustre is a trademark of Sun Microsystems, Inc.
  37  */
  38
  39 #ifndef EXPORT_SYMTAB
  40 # define EXPORT_SYMTAB
  41 #endif
  42 #define DEBUG_SUBSYSTEM S_LQUOTA
  43
  44 #ifdef __KERNEL__
  45 # include <linux/version.h>
  46 # include <linux/module.h>
  47 # include <linux/init.h>
  48 # include <linux/fs.h>
  49 # include <linux/jbd.h>
  50 # include <linux/smp_lock.h>
  51 # include <linux/buffer_head.h>
  52 # include <linux/workqueue.h>
  53 # include <linux/mount.h>
  54 #else /* __KERNEL__ */
  55 # include <liblustre.h>
  56 #endif
  57
  58 #include <obd_class.h>
  59 #include <lustre_mds.h>
  60 #include <lustre_dlm.h>
  61 #include <lustre_cfg.h>
  62 #include <obd_ost.h>
  63 #include <lustre_fsfilt.h>
  64 #include <lustre_quota.h>
  65 #include <lprocfs_status.h>
  66 #include "quota_internal.h"
  67
  68 #ifdef __KERNEL__
  69
  70 #ifdef HAVE_QUOTA_SUPPORT
  71
  72 static cfs_time_t last_print = 0;
  73 static cfs_spinlock_t last_print_lock = CFS_SPIN_LOCK_UNLOCKED;
  74
  75 static int filter_quota_setup(struct obd_device *obd)
  76 {
  77         int rc = 0;
  78         struct obd_device_target *obt = &obd->u.obt;
  79         ENTRY;
  80
  81         cfs_init_rwsem(&obt->obt_rwsem);
  82         obt->obt_qfmt = LUSTRE_QUOTA_V2;
  83         cfs_sema_init(&obt->obt_quotachecking, 1);
  84         rc = qctxt_init(obd, NULL);
  85         if (rc)
  86                 CERROR("initialize quota context failed! (rc:%d)\n", rc);
  87
  88         RETURN(rc);
  89 }
  90
  91 static int filter_quota_cleanup(struct obd_device *obd)
  92 {
  93         ENTRY;
  94         qctxt_cleanup(&obd->u.obt.obt_qctxt, 0);
  95         RETURN(0);
  96 }
  97
  98 static int filter_quota_setinfo(struct obd_device *obd, void *data)
  99 {
 100         struct obd_export *exp = data;
 101         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 102         struct obd_import *imp = exp->exp_imp_reverse;
 103         ENTRY;
 104
 105         LASSERT(imp != NULL);
 106
 107         /* setup the quota context import */
 108         cfs_spin_lock(&qctxt->lqc_lock);
 109         if (qctxt->lqc_import != NULL) {
 110                 cfs_spin_unlock(&qctxt->lqc_lock);
 111                 if (qctxt->lqc_import == imp)
 112                         CDEBUG(D_WARNING, "%s: lqc_import(%p) of obd(%p) was "
 113                                "activated already.\n", obd->obd_name, imp, obd);
 114                 else
 115                         CERROR("%s: lqc_import(%p:%p) of obd(%p) was "
 116                                "activated by others.\n", obd->obd_name,
 117                                qctxt->lqc_import, imp, obd);
 118         } else {
 119                 qctxt->lqc_import = imp;
 120                 /* make imp's connect flags equal relative exp's connect flags
 121                  * adding it to avoid the scan export list */
 122                 imp->imp_connect_data.ocd_connect_flags |=
 123                                 (exp->exp_connect_flags &
 124                                  (OBD_CONNECT_QUOTA64 | OBD_CONNECT_CHANGE_QS));
 125                 cfs_spin_unlock(&qctxt->lqc_lock);
 126                 CDEBUG(D_QUOTA, "%s: lqc_import(%p) of obd(%p) is reactivated "
 127                        "now.\n", obd->obd_name, imp, obd);
 128
 129                 cfs_waitq_signal(&qctxt->lqc_wait_for_qmaster);
 130                 /* start quota slave recovery thread. (release high limits) */
 131                 qslave_start_recovery(obd, qctxt);
 132         }
 133         RETURN(0);
 134 }
 135
 136 static int filter_quota_clearinfo(struct obd_export *exp, struct obd_device *obd)
 137 {
 138         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 139         struct obd_import *imp = exp->exp_imp_reverse;
 140         ENTRY;
 141
 142         /* lquota may be not set up before destroying export, b=14896 */
 143         if (!obd->obd_set_up)
 144                 RETURN(0);
 145
 146         if (unlikely(imp == NULL))
 147                 RETURN(0);
 148
 149         /* when exp->exp_imp_reverse is destroyed, the corresponding lqc_import
 150          * should be invalid b=12374 */
 151         cfs_spin_lock(&qctxt->lqc_lock);
 152         if (qctxt->lqc_import == imp) {
 153                 qctxt->lqc_import = NULL;
 154                 cfs_spin_unlock(&qctxt->lqc_lock);
 155                 CDEBUG(D_QUOTA, "%s: lqc_import(%p) of obd(%p) is invalid now.\n",
 156                        obd->obd_name, imp, obd);
 157                 ptlrpc_cleanup_imp(imp);
 158                 dqacq_interrupt(qctxt);
 159         } else {
 160                 cfs_spin_unlock(&qctxt->lqc_lock);
 161         }
 162         RETURN(0);
 163 }
 164
 165 static int filter_quota_enforce(struct obd_device *obd, unsigned int ignore)
 166 {
 167         ENTRY;
 168
 169         if (!ll_sb_any_quota_active(obd->u.obt.obt_sb))
 170                 RETURN(0);
 171
 172         if (ignore) {
 173                 CDEBUG(D_QUOTA, "blocks will be written with ignoring quota.\n");
 174                 cfs_cap_raise(CFS_CAP_SYS_RESOURCE);
 175         } else {
 176                 cfs_cap_lower(CFS_CAP_SYS_RESOURCE);
 177         }
 178
 179         RETURN(0);
 180 }
 181
 182 #define GET_OA_ID(flag, oa) (flag == USRQUOTA ? oa->o_uid : oa->o_gid)
 183 static int filter_quota_getflag(struct obd_device *obd, struct obdo *oa)
 184 {
 185         struct obd_device_target *obt = &obd->u.obt;
 186         struct lustre_quota_ctxt *qctxt = &obt->obt_qctxt;
 187         int err, cnt, rc = 0;
 188         struct obd_quotactl *oqctl;
 189         ENTRY;
 190
 191         if (!ll_sb_any_quota_active(obt->obt_sb))
 192                 RETURN(0);
 193
 194         OBD_ALLOC_PTR(oqctl);
 195         if (!oqctl)
 196                 RETURN(-ENOMEM);
 197
 198         /* set over quota flags for a uid/gid */
 199         oa->o_valid |= OBD_MD_FLUSRQUOTA | OBD_MD_FLGRPQUOTA;
 200         oa->o_flags &= ~(OBD_FL_NO_USRQUOTA | OBD_FL_NO_GRPQUOTA);
 201
 202         for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
 203                 struct lustre_qunit_size *lqs = NULL;
 204
 205                 lqs = quota_search_lqs(LQS_KEY(cnt, GET_OA_ID(cnt, oa)),
 206                                        qctxt, 0);
 207                 if (IS_ERR(lqs)) {
 208                         rc = PTR_ERR(lqs);
 209                         CDEBUG(D_QUOTA, "search lqs for %s %d failed, "
 210                                "(rc = %d)\n",
 211                                cnt == USRQUOTA ? "user" : "group",
 212                                GET_OA_ID(cnt, oa), rc);
 213                         break;
 214                 } else if (lqs == NULL) {
 215                         /* continue to check group quota if the file's owner
 216                          * doesn't have quota limit. LU-530 */
 217                         continue;
 218                 } else {
 219                         cfs_spin_lock(&lqs->lqs_lock);
 220                         if (lqs->lqs_bunit_sz <= qctxt->lqc_sync_blk) {
 221                                 oa->o_flags |= (cnt == USRQUOTA) ?
 222                                         OBD_FL_NO_USRQUOTA : OBD_FL_NO_GRPQUOTA;
 223                                 cfs_spin_unlock(&lqs->lqs_lock);
 224                                 CDEBUG(D_QUOTA, "set sync flag: bunit(%lu), "
 225                                        "sync_blk(%d)\n", lqs->lqs_bunit_sz,
 226                                        qctxt->lqc_sync_blk);
 227                                 /* this is for quota_search_lqs */
 228                                 lqs_putref(lqs);
 229                                 continue;
 230                         }
 231                         cfs_spin_unlock(&lqs->lqs_lock);
 232                         /* this is for quota_search_lqs */
 233                         lqs_putref(lqs);
 234                 }
 235
 236                 memset(oqctl, 0, sizeof(*oqctl));
 237
 238                 oqctl->qc_cmd = Q_GETQUOTA;
 239                 oqctl->qc_type = cnt;
 240                 oqctl->qc_id = (cnt == USRQUOTA) ? oa->o_uid : oa->o_gid;
 241                 err = fsfilt_quotactl(obd, obt->obt_sb, oqctl);
 242                 if (err) {
 243                         if (!rc)
 244                                 rc = err;
 245                         oa->o_valid &= ~((cnt == USRQUOTA) ? OBD_MD_FLUSRQUOTA :
 246                                                              OBD_MD_FLGRPQUOTA);
 247                         CDEBUG(D_QUOTA, "fsfilt getquota for %s %d failed, "
 248                                "(rc = %d)\n",
 249                                cnt == USRQUOTA ? "user" : "group",
 250                                cnt == USRQUOTA ? oa->o_uid : oa->o_gid, err);
 251                         continue;
 252                 }
 253
 254                 if (oqctl->qc_dqblk.dqb_bhardlimit &&
 255                    (toqb(oqctl->qc_dqblk.dqb_curspace) >=
 256                     oqctl->qc_dqblk.dqb_bhardlimit)) {
 257                         oa->o_flags |= (cnt == USRQUOTA) ?
 258                                 OBD_FL_NO_USRQUOTA : OBD_FL_NO_GRPQUOTA;
 259                         CDEBUG(D_QUOTA, "out of quota for %s %d\n",
 260                                cnt == USRQUOTA ? "user" : "group",
 261                                cnt == USRQUOTA ? oa->o_uid : oa->o_gid);
 262                 }
 263         }
 264         OBD_FREE_PTR(oqctl);
 265         RETURN(rc);
 266 }
 267
 268 /**
 269  * check whether the left quota of certain uid and gid can satisfy a block_write
 270  * or inode_create rpc. When need to acquire quota, return QUOTA_RET_ACQUOTA
 271  */
 272 static int quota_check_common(struct obd_device *obd, const unsigned int id[],
 273                               int pending[], int count, int cycle, int isblk,
 274                               struct inode *inode, int frags)
 275 {
 276         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 277         int i;
 278         struct qunit_data qdata[MAXQUOTAS];
 279         int mb = 0;
 280         int rc = 0, rc2[2] = { 0, 0 };
 281         ENTRY;
 282
 283         cfs_spin_lock(&qctxt->lqc_lock);
 284         if (!qctxt->lqc_valid){
 285                 cfs_spin_unlock(&qctxt->lqc_lock);
 286                 RETURN(rc);
 287         }
 288         cfs_spin_unlock(&qctxt->lqc_lock);
 289
 290         for (i = 0; i < MAXQUOTAS; i++) {
 291                 struct lustre_qunit_size *lqs = NULL;
 292
 293                 qdata[i].qd_id = id[i];
 294                 qdata[i].qd_flags = i;
 295                 if (isblk)
 296                         QDATA_SET_BLK(&qdata[i]);
 297                 qdata[i].qd_count = 0;
 298
 299                 /* ignore root user */
 300                 if (qdata[i].qd_id == 0 && !QDATA_IS_GRP(&qdata[i]))
 301                         continue;
 302
 303                 lqs = quota_search_lqs(LQS_KEY(i, id[i]), qctxt, 0);
 304                 if (lqs == NULL || IS_ERR(lqs))
 305                         continue;
 306
 307                 if (IS_ERR(lqs)) {
 308                         CERROR("can not find lqs for check_common: "
 309                                "[id %u] [%c] [isblk %d] [count %d] [rc %ld]\n",
 310                                id[i], i % 2 ? 'g': 'u', isblk, count,
 311                                PTR_ERR(lqs));
 312                         RETURN(PTR_ERR(lqs));
 313                 }
 314
 315                 rc2[i] = compute_remquota(obd, qctxt, &qdata[i], isblk);
 316                 cfs_spin_lock(&lqs->lqs_lock);
 317                 if (!cycle) {
 318                         if (isblk) {
 319                                 pending[i] = count * CFS_PAGE_SIZE;
 320                                 /* in order to complete this write, we need extra
 321                                  * meta blocks. This function can get it through
 322                                  * data needed to be written b=16542 */
 323                                 if (inode) {
 324                                         mb = pending[i];
 325                                         rc = fsfilt_get_mblk(obd, qctxt->lqc_sb,
 326                                                              &mb, inode,
 327                                                              frags);
 328                                         if (rc)
 329                                                 CERROR("%s: can't get extra "
 330                                                        "meta blocks\n",
 331                                                        obd->obd_name);
 332                                         else
 333                                                 pending[i] += mb;
 334                                 }
 335                                 LASSERTF(pending[i] >= 0, "pending is not valid"
 336                                          ", count=%d, mb=%d\n", count, mb);
 337                                 lqs->lqs_bwrite_pending += pending[i];
 338                         } else {
 339                                 pending[i] = count;
 340                                 lqs->lqs_iwrite_pending += pending[i];
 341                         }
 342                 }
 343
 344                 /* if xx_rec < 0, that means quota are releasing,
 345                  * and it may return before we use quota. So if
 346                  * we find this situation, we assuming it has
 347                  * returned b=18491 */
 348                 if (isblk && lqs->lqs_blk_rec < 0) {
 349                         if (qdata[i].qd_count < -lqs->lqs_blk_rec)
 350                                 qdata[i].qd_count = 0;
 351                         else
 352                                 qdata[i].qd_count += lqs->lqs_blk_rec;
 353                 }
 354                 if (!isblk && lqs->lqs_ino_rec < 0) {
 355                         if (qdata[i].qd_count < -lqs->lqs_ino_rec)
 356                                 qdata[i].qd_count = 0;
 357                         else
 358                                 qdata[i].qd_count += lqs->lqs_ino_rec;
 359                 }
 360
 361                 CDEBUG(D_QUOTA, "[id %u] [%c] [isblk %d] [count %d]"
 362                        " [lqs pending: %lu] [qd_count: "LPU64"] [metablocks: %d]"
 363                        " [pending: %d]\n", id[i], i % 2 ? 'g': 'u', isblk, count,
 364                        isblk ? lqs->lqs_bwrite_pending : lqs->lqs_iwrite_pending,
 365                        qdata[i].qd_count, mb, pending[i]);
 366                 if (rc2[i] == QUOTA_RET_OK) {
 367                         if (isblk && qdata[i].qd_count < lqs->lqs_bwrite_pending)
 368                                 rc2[i] = QUOTA_RET_ACQUOTA;
 369                         if (!isblk && qdata[i].qd_count <
 370                             lqs->lqs_iwrite_pending)
 371                                 rc2[i] = QUOTA_RET_ACQUOTA;
 372                 }
 373
 374                 cfs_spin_unlock(&lqs->lqs_lock);
 375
 376                 if (lqs->lqs_blk_rec  < 0 &&
 377                     qdata[i].qd_count <
 378                     lqs->lqs_bwrite_pending - lqs->lqs_blk_rec - mb)
 379                         OBD_FAIL_TIMEOUT(OBD_FAIL_QUOTA_DELAY_REL, 5);
 380
 381                 /* When cycle is zero, lqs_*_pending will be changed. We will
 382                  * get reference of the lqs here and put reference of lqs in
 383                  * quota_pending_commit b=14784 */
 384                 if (!cycle)
 385                         lqs_getref(lqs);
 386
 387                 /* this is for quota_search_lqs */
 388                 lqs_putref(lqs);
 389         }
 390
 391         if (rc2[0] == QUOTA_RET_ACQUOTA || rc2[1] == QUOTA_RET_ACQUOTA)
 392                 RETURN(QUOTA_RET_ACQUOTA);
 393         else
 394                 RETURN(rc);
 395 }
 396
 397 int quota_is_set(struct obd_device *obd, const unsigned int id[], int flag)
 398 {
 399         struct lustre_qunit_size *lqs;
 400         int i, q_set = 0;
 401
 402         if (!ll_sb_any_quota_active(obd->u.obt.obt_qctxt.lqc_sb))
 403                 RETURN(0);
 404
 405         for (i = 0; i < MAXQUOTAS; i++) {
 406                 lqs = quota_search_lqs(LQS_KEY(i, id[i]),
 407                                        &obd->u.obt.obt_qctxt, 0);
 408                 if (lqs && !IS_ERR(lqs)) {
 409                         if (lqs->lqs_flags & flag)
 410                                 q_set = 1;
 411                         lqs_putref(lqs);
 412                 }
 413         }
 414
 415         return q_set;
 416 }
 417
 418 static int quota_chk_acq_common(struct obd_device *obd, struct obd_export *exp,
 419                                 const unsigned int id[], int pending[],
 420                                 int count, quota_acquire acquire,
 421                                 struct obd_trans_info *oti, int isblk,
 422                                 struct inode *inode, int frags)
 423 {
 424         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 425         struct timeval work_start;
 426         struct timeval work_end;
 427         long timediff;
 428         struct l_wait_info lwi = { 0 };
 429         int rc = 0, cycle = 0, count_err = 1;
 430         ENTRY;
 431
 432         if (!quota_is_set(obd, id, isblk ? QB_SET : QI_SET))
 433                 RETURN(0);
 434
 435         if (isblk && (exp->exp_failed || exp->exp_abort_active_req))
 436                 /* If the client has been evicted or if it
 437                  * timed out and tried to reconnect already,
 438                  * abort the request immediately */
 439                 RETURN(-ENOTCONN);
 440
 441         CDEBUG(D_QUOTA, "check quota for %s\n", obd->obd_name);
 442         pending[USRQUOTA] = pending[GRPQUOTA] = 0;
 443         /* Unfortunately, if quota master is too busy to handle the
 444          * pre-dqacq in time and quota hash on ost is used up, we
 445          * have to wait for the completion of in flight dqacq/dqrel,
 446          * in order to get enough quota for write b=12588 */
 447         cfs_gettimeofday(&work_start);
 448         while ((rc = quota_check_common(obd, id, pending, count, cycle, isblk,
 449                                         inode, frags)) &
 450                QUOTA_RET_ACQUOTA) {
 451
 452                 cfs_spin_lock(&qctxt->lqc_lock);
 453                 if (!qctxt->lqc_import && oti) {
 454                         cfs_spin_unlock(&qctxt->lqc_lock);
 455                         LASSERT(oti->oti_thread);
 456                         /* The recovery thread doesn't have watchdog
 457                          * attached. LU-369 */
 458                         if (oti->oti_thread->t_watchdog)
 459                                 lc_watchdog_disable(oti->oti_thread->\
 460                                                 t_watchdog);
 461                         CDEBUG(D_QUOTA, "sleep for quota master\n");
 462                         l_wait_event(qctxt->lqc_wait_for_qmaster, check_qm(qctxt),
 463                                      &lwi);
 464                         CDEBUG(D_QUOTA, "wake up when quota master is back\n");
 465                         if (oti->oti_thread->t_watchdog)
 466                                 lc_watchdog_touch(oti->oti_thread->t_watchdog,
 467                                        CFS_GET_TIMEOUT(oti->oti_thread->t_svc));
 468                 } else {
 469                         cfs_spin_unlock(&qctxt->lqc_lock);
 470                 }
 471
 472                 cycle++;
 473                 if (isblk)
 474                         OBD_FAIL_TIMEOUT(OBD_FAIL_OST_HOLD_WRITE_RPC, 90);
 475                 /* after acquire(), we should run quota_check_common again
 476                  * so that we confirm there are enough quota to finish write */
 477                 rc = acquire(obd, id, oti, isblk);
 478
 479                 /* please reference to dqacq_completion for the below */
 480                 /* a new request is finished, try again */
 481                 if (rc == QUOTA_REQ_RETURNED) {
 482                         CDEBUG(D_QUOTA, "finish a quota req, try again\n");
 483                         continue;
 484                 }
 485
 486                 /* it is out of quota already */
 487                 if (rc == -EDQUOT) {
 488                         CDEBUG(D_QUOTA, "out of quota,  return -EDQUOT\n");
 489                         break;
 490                 }
 491
 492                 /* Related quota has been disabled by master, but enabled by
 493                  * slave, do not try again. */
 494                 if (unlikely(rc == -ESRCH)) {
 495                         CERROR("mismatched quota configuration, stop try.\n");
 496                         break;
 497                 }
 498
 499                 if (isblk && (exp->exp_failed || exp->exp_abort_active_req))
 500                         /* The client has been evicted or tried to
 501                          * to reconnect already, abort the request */
 502                         RETURN(-ENOTCONN);
 503
 504                 /* -EBUSY and others, wait a second and try again */
 505                 if (rc < 0) {
 506                         cfs_waitq_t        waitq;
 507                         struct l_wait_info lwi;
 508
 509                         if (oti && oti->oti_thread && oti->oti_thread->t_watchdog)
 510                                 lc_watchdog_touch(oti->oti_thread->t_watchdog,
 511                                        CFS_GET_TIMEOUT(oti->oti_thread->t_svc));
 512                         CDEBUG(D_QUOTA, "rc: %d, count_err: %d\n", rc,
 513                                count_err++);
 514
 515                         cfs_waitq_init(&waitq);
 516                         lwi = LWI_TIMEOUT(cfs_time_seconds(min(cycle, 10)), NULL,
 517                                           NULL);
 518                         l_wait_event(waitq, 0, &lwi);
 519                 }
 520
 521                 if (rc < 0 || cycle % 10 == 0) {
 522                         cfs_spin_lock(&last_print_lock);
 523                         if (last_print == 0 ||
 524                             cfs_time_before((last_print + cfs_time_seconds(30)),
 525                                             cfs_time_current())) {
 526                                 last_print = cfs_time_current();
 527                                 cfs_spin_unlock(&last_print_lock);
 528                                 CWARN("still haven't managed to acquire quota "
 529                                       "space from the quota master after %d "
 530                                       "retries (err=%d, rc=%d)\n",
 531                                       cycle, count_err - 1, rc);
 532                         } else {
 533                                 cfs_spin_unlock(&last_print_lock);
 534                         }
 535                 }
 536
 537                 CDEBUG(D_QUOTA, "recheck quota with rc: %d, cycle: %d\n", rc,
 538                        cycle);
 539         }
 540         cfs_gettimeofday(&work_end);
 541         timediff = cfs_timeval_sub(&work_end, &work_start, NULL);
 542         lprocfs_counter_add(qctxt->lqc_stats,
 543                             isblk ? LQUOTA_WAIT_FOR_CHK_BLK :
 544                                     LQUOTA_WAIT_FOR_CHK_INO,
 545                             timediff);
 546
 547         if (rc > 0)
 548                 rc = 0;
 549         RETURN(rc);
 550 }
 551
 552 /**
 553  * when a block_write or inode_create rpc is finished, adjust the record for
 554  * pending blocks and inodes
 555  */
 556 static int quota_pending_commit(struct obd_device *obd, const unsigned int id[],
 557                                 int pending[], int isblk)
 558 {
 559         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 560         struct timeval work_start;
 561         struct timeval work_end;
 562         long timediff;
 563         int i;
 564         struct qunit_data qdata[MAXQUOTAS];
 565         ENTRY;
 566
 567         CDEBUG(D_QUOTA, "commit pending quota for  %s\n", obd->obd_name);
 568         CLASSERT(MAXQUOTAS < 4);
 569         if (!ll_sb_any_quota_active(qctxt->lqc_sb))
 570                 RETURN(0);
 571
 572         cfs_gettimeofday(&work_start);
 573         for (i = 0; i < MAXQUOTAS; i++) {
 574                 struct lustre_qunit_size *lqs = NULL;
 575
 576                 LASSERT(pending[i] >= 0);
 577                 if (pending[i] == 0)
 578                         continue;
 579
 580                 qdata[i].qd_id = id[i];
 581                 qdata[i].qd_flags = i;
 582                 if (isblk)
 583                         QDATA_SET_BLK(&qdata[i]);
 584                 qdata[i].qd_count = 0;
 585
 586                 if (qdata[i].qd_id == 0 && !QDATA_IS_GRP(&qdata[i]))
 587                         continue;
 588
 589                 lqs = quota_search_lqs(LQS_KEY(i, qdata[i].qd_id), qctxt, 0);
 590                 if (lqs == NULL || IS_ERR(lqs)) {
 591                         CERROR("can not find lqs for pending_commit: "
 592                                "[id %u] [%c] [pending %u] [isblk %d] (rc %ld), "
 593                                "maybe cause unexpected lqs refcount error!\n",
 594                                id[i], i ? 'g': 'u', pending[i], isblk,
 595                                lqs ? PTR_ERR(lqs) : -1);
 596                         continue;
 597                 }
 598
 599                 cfs_spin_lock(&lqs->lqs_lock);
 600                 if (isblk) {
 601                         LASSERTF(lqs->lqs_bwrite_pending >= pending[i],
 602                                  "there are too many blocks! [id %u] [%c] "
 603                                  "[bwrite_pending %lu] [pending %u]\n",
 604                                  id[i], i % 2 ? 'g' : 'u',
 605                                  lqs->lqs_bwrite_pending, pending[i]);
 606
 607                         lqs->lqs_bwrite_pending -= pending[i];
 608                 } else {
 609                         LASSERTF(lqs->lqs_iwrite_pending >= pending[i],
 610                                 "there are too many files! [id %u] [%c] "
 611                                 "[iwrite_pending %lu] [pending %u]\n",
 612                                 id[i], i % 2 ? 'g' : 'u',
 613                                 lqs->lqs_iwrite_pending, pending[i]);
 614
 615                         lqs->lqs_iwrite_pending -= pending[i];
 616                 }
 617                 CDEBUG(D_QUOTA, "%s: lqs_pending=%lu pending[%d]=%d isblk=%d\n",
 618                        obd->obd_name,
 619                        isblk ? lqs->lqs_bwrite_pending : lqs->lqs_iwrite_pending,
 620                        i, pending[i], isblk);
 621                 cfs_spin_unlock(&lqs->lqs_lock);
 622
 623                 /* for quota_search_lqs in pending_commit */
 624                 lqs_putref(lqs);
 625                 /* for quota_search_lqs in quota_check */
 626                 lqs_putref(lqs);
 627         }
 628         cfs_gettimeofday(&work_end);
 629         timediff = cfs_timeval_sub(&work_end, &work_start, NULL);
 630         lprocfs_counter_add(qctxt->lqc_stats,
 631                             isblk ? LQUOTA_WAIT_FOR_COMMIT_BLK :
 632                                     LQUOTA_WAIT_FOR_COMMIT_INO,
 633                             timediff);
 634
 635         RETURN(0);
 636 }
 637
 638 static int mds_quota_init(void)
 639 {
 640         return lustre_dquot_init();
 641 }
 642
 643 static int mds_quota_exit(void)
 644 {
 645         lustre_dquot_exit();
 646         return 0;
 647 }
 648
 649 static int mds_quota_setup(struct obd_device *obd)
 650 {
 651         struct obd_device_target *obt = &obd->u.obt;
 652         struct mds_obd *mds = &obd->u.mds;
 653         int rc;
 654         ENTRY;
 655
 656         if (unlikely(mds->mds_quota)) {
 657                 CWARN("try to reinitialize quota context!\n");
 658                 RETURN(0);
 659         }
 660
 661         cfs_init_rwsem(&obt->obt_rwsem);
 662         obt->obt_qfmt = LUSTRE_QUOTA_V2;
 663         mds->mds_quota_info.qi_version = LUSTRE_QUOTA_V2;
 664         cfs_sema_init(&obt->obt_quotachecking, 1);
 665         /* initialize quota master and quota context */
 666         cfs_init_rwsem(&mds->mds_qonoff_sem);
 667         rc = qctxt_init(obd, dqacq_handler);
 668         if (rc) {
 669                 CERROR("%s: initialize quota context failed! (rc:%d)\n",
 670                        obd->obd_name, rc);
 671                 RETURN(rc);
 672         }
 673         mds->mds_quota = 1;
 674         RETURN(rc);
 675 }
 676
 677 static int mds_quota_cleanup(struct obd_device *obd)
 678 {
 679         ENTRY;
 680         if (unlikely(!obd->u.mds.mds_quota))
 681                 RETURN(0);
 682
 683         qctxt_cleanup(&obd->u.obt.obt_qctxt, 0);
 684         RETURN(0);
 685 }
 686
 687 static int mds_quota_setinfo(struct obd_device *obd, void *data)
 688 {
 689         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 690         ENTRY;
 691
 692         if (unlikely(!obd->u.mds.mds_quota))
 693                 RETURN(0);
 694
 695         if (data != NULL)
 696                 QUOTA_MASTER_READY(qctxt);
 697         else
 698                 QUOTA_MASTER_UNREADY(qctxt);
 699         RETURN(0);
 700 }
 701
 702 static int mds_quota_fs_cleanup(struct obd_device *obd)
 703 {
 704         struct mds_obd *mds = &obd->u.mds;
 705         struct obd_quotactl oqctl;
 706         ENTRY;
 707
 708         if (unlikely(!mds->mds_quota))
 709                 RETURN(0);
 710
 711         mds->mds_quota = 0;
 712         memset(&oqctl, 0, sizeof(oqctl));
 713         oqctl.qc_type = UGQUOTA;
 714
 715         cfs_down_write(&mds->mds_qonoff_sem);
 716         mds_admin_quota_off(obd, &oqctl);
 717         cfs_up_write(&mds->mds_qonoff_sem);
 718         RETURN(0);
 719 }
 720
 721 static int quota_acquire_common(struct obd_device *obd, const unsigned int id[],
 722                                 struct obd_trans_info *oti, int isblk)
 723 {
 724         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 725         int rc;
 726         ENTRY;
 727
 728         rc = qctxt_adjust_qunit(obd, qctxt, id, isblk, 1, oti);
 729         RETURN(rc);
 730 }
 731
 732 #endif /* HAVE_QUOTA_SUPPORT */
 733 #endif /* __KERNEL__ */
 734
 735 #ifdef __KERNEL__
 736 #ifdef HAVE_QUOTA_SUPPORT
 737 quota_interface_t mds_quota_interface = {
 738         .quota_init     = mds_quota_init,
 739         .quota_exit     = mds_quota_exit,
 740         .quota_setup    = mds_quota_setup,
 741         .quota_cleanup  = mds_quota_cleanup,
 742         .quota_check    = target_quota_check,
 743         .quota_ctl      = mds_quota_ctl,
 744         .quota_setinfo  = mds_quota_setinfo,
 745         .quota_fs_cleanup = mds_quota_fs_cleanup,
 746         .quota_recovery = mds_quota_recovery,
 747         .quota_adjust   = mds_quota_adjust,
 748         .quota_chkquota = quota_chk_acq_common,
 749         .quota_acquire  = quota_acquire_common,
 750         .quota_pending_commit = quota_pending_commit,
 751 };
 752
 753 quota_interface_t filter_quota_interface = {
 754         .quota_setup    = filter_quota_setup,
 755         .quota_cleanup  = filter_quota_cleanup,
 756         .quota_check    = target_quota_check,
 757         .quota_ctl      = filter_quota_ctl,
 758         .quota_setinfo  = filter_quota_setinfo,
 759         .quota_clearinfo = filter_quota_clearinfo,
 760         .quota_enforce  = filter_quota_enforce,
 761         .quota_getflag  = filter_quota_getflag,
 762         .quota_acquire  = quota_acquire_common,
 763         .quota_adjust   = filter_quota_adjust,
 764         .quota_chkquota = quota_chk_acq_common,
 765         .quota_adjust_qunit   = filter_quota_adjust_qunit,
 766         .quota_pending_commit = quota_pending_commit,
 767 };
 768 #endif
 769 #endif /* __KERNEL__ */
 770
 771 #ifdef __KERNEL__
 772
 773 cfs_proc_dir_entry_t *lquota_type_proc_dir = NULL;
 774
 775 static int __init init_lustre_quota(void)
 776 {
 777 #ifdef HAVE_QUOTA_SUPPORT
 778         int rc = 0;
 779
 780         lquota_type_proc_dir = lprocfs_register(OBD_LQUOTA_DEVICENAME,
 781                                                 proc_lustre_root,
 782                                                 NULL, NULL);
 783         if (IS_ERR(lquota_type_proc_dir)) {
 784                 CERROR("LProcFS failed in lquota-init\n");
 785                 rc = PTR_ERR(lquota_type_proc_dir);
 786                 return rc;
 787         }
 788
 789         rc = qunit_cache_init();
 790         if (rc)
 791                 return rc;
 792
 793         PORTAL_SYMBOL_REGISTER(filter_quota_interface);
 794         PORTAL_SYMBOL_REGISTER(mds_quota_interface);
 795 #endif
 796         return 0;
 797 }
 798
 799 static void /*__exit*/ exit_lustre_quota(void)
 800 {
 801 #ifdef HAVE_QUOTA_SUPPORT
 802         PORTAL_SYMBOL_UNREGISTER(filter_quota_interface);
 803         PORTAL_SYMBOL_UNREGISTER(mds_quota_interface);
 804
 805         qunit_cache_cleanup();
 806
 807         if (lquota_type_proc_dir)
 808                 lprocfs_remove(&lquota_type_proc_dir);
 809 #endif
 810 }
 811
 812 MODULE_AUTHOR("Sun Microsystems, Inc. <http://www.lustre.org/>");
 813 MODULE_DESCRIPTION("Lustre Quota");
 814 MODULE_LICENSE("GPL");
 815
 816 cfs_module(lquota, "1.0.0", init_lustre_quota, exit_lustre_quota);
 817
 818 #ifdef HAVE_QUOTA_SUPPORT
 819 EXPORT_SYMBOL(mds_quota_interface);
 820 EXPORT_SYMBOL(filter_quota_interface);
 821 #endif
 822 #endif /* __KERNEL */