lustre/quota/quota_interface.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2011, 2012, Whamcloud, Inc.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  */
  36
  37 #define DEBUG_SUBSYSTEM S_LQUOTA
  38
  39 #ifdef __KERNEL__
  40 # include <linux/version.h>
  41 # include <linux/module.h>
  42 # include <linux/init.h>
  43 # include <linux/fs.h>
  44 # include <linux/jbd.h>
  45 # include <linux/smp_lock.h>
  46 # include <linux/buffer_head.h>
  47 # include <linux/workqueue.h>
  48 # include <linux/mount.h>
  49 #else /* __KERNEL__ */
  50 # include <liblustre.h>
  51 #endif
  52
  53 #include <obd_class.h>
  54 #include <lustre_mds.h>
  55 #include <lustre_dlm.h>
  56 #include <lustre_cfg.h>
  57 #include <obd_ost.h>
  58 #include <lustre_fsfilt.h>
  59 #include <lustre_quota.h>
  60 #include <lprocfs_status.h>
  61 #include "quota_internal.h"
  62
  63 #ifdef __KERNEL__
  64
  65 static cfs_time_t last_print = 0;
  66 static cfs_spinlock_t last_print_lock = CFS_SPIN_LOCK_UNLOCKED;
  67
  68 static int filter_quota_setup(struct obd_device *obd)
  69 {
  70         int rc = 0;
  71         struct obd_device_target *obt = &obd->u.obt;
  72         ENTRY;
  73
  74         cfs_init_rwsem(&obt->obt_rwsem);
  75         obt->obt_qfmt = LUSTRE_QUOTA_V2;
  76         cfs_sema_init(&obt->obt_quotachecking, 1);
  77         rc = qctxt_init(obd, NULL);
  78         if (rc)
  79                 CERROR("initialize quota context failed! (rc:%d)\n", rc);
  80
  81         RETURN(rc);
  82 }
  83
  84 static int filter_quota_cleanup(struct obd_device *obd)
  85 {
  86         ENTRY;
  87         qctxt_cleanup(&obd->u.obt.obt_qctxt, 0);
  88         RETURN(0);
  89 }
  90
  91 static int filter_quota_setinfo(struct obd_device *obd, void *data)
  92 {
  93         struct obd_export *exp = data;
  94         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
  95         struct obd_import *imp = exp->exp_imp_reverse;
  96         ENTRY;
  97
  98         LASSERT(imp != NULL);
  99
 100         /* setup the quota context import */
 101         cfs_spin_lock(&qctxt->lqc_lock);
 102         if (qctxt->lqc_import != NULL) {
 103                 cfs_spin_unlock(&qctxt->lqc_lock);
 104                 if (qctxt->lqc_import == imp)
 105                         CDEBUG(D_WARNING, "%s: lqc_import(%p) of obd(%p) was "
 106                                "activated already.\n", obd->obd_name, imp, obd);
 107                 else
 108                         CERROR("%s: lqc_import(%p:%p) of obd(%p) was "
 109                                "activated by others.\n", obd->obd_name,
 110                                qctxt->lqc_import, imp, obd);
 111         } else {
 112                 qctxt->lqc_import = imp;
 113                 /* make imp's connect flags equal relative exp's connect flags
 114                  * adding it to avoid the scan export list */
 115                 imp->imp_connect_data.ocd_connect_flags |=
 116                                 (exp->exp_connect_flags &
 117                                  (OBD_CONNECT_QUOTA64 | OBD_CONNECT_CHANGE_QS));
 118                 cfs_spin_unlock(&qctxt->lqc_lock);
 119                 CDEBUG(D_QUOTA, "%s: lqc_import(%p) of obd(%p) is reactivated "
 120                        "now.\n", obd->obd_name, imp, obd);
 121
 122                 cfs_waitq_signal(&qctxt->lqc_wait_for_qmaster);
 123                 /* start quota slave recovery thread. (release high limits) */
 124                 qslave_start_recovery(obd, qctxt);
 125         }
 126         RETURN(0);
 127 }
 128
 129 static int filter_quota_clearinfo(struct obd_export *exp, struct obd_device *obd)
 130 {
 131         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 132         struct obd_import *imp = exp->exp_imp_reverse;
 133         ENTRY;
 134
 135         /* lquota may be not set up before destroying export, b=14896 */
 136         if (!obd->obd_set_up)
 137                 RETURN(0);
 138
 139         if (unlikely(imp == NULL))
 140                 RETURN(0);
 141
 142         /* when exp->exp_imp_reverse is destroyed, the corresponding lqc_import
 143          * should be invalid b=12374 */
 144         cfs_spin_lock(&qctxt->lqc_lock);
 145         if (qctxt->lqc_import == imp) {
 146                 qctxt->lqc_import = NULL;
 147                 cfs_spin_unlock(&qctxt->lqc_lock);
 148                 CDEBUG(D_QUOTA, "%s: lqc_import(%p) of obd(%p) is invalid now.\n",
 149                        obd->obd_name, imp, obd);
 150                 ptlrpc_cleanup_imp(imp);
 151                 dqacq_interrupt(qctxt);
 152         } else {
 153                 cfs_spin_unlock(&qctxt->lqc_lock);
 154         }
 155         RETURN(0);
 156 }
 157
 158 static int filter_quota_enforce(struct obd_device *obd, unsigned int ignore)
 159 {
 160         ENTRY;
 161
 162         if (!ll_sb_any_quota_active(obd->u.obt.obt_sb))
 163                 RETURN(0);
 164
 165         if (ignore) {
 166                 CDEBUG(D_QUOTA, "blocks will be written with ignoring quota.\n");
 167                 cfs_cap_raise(CFS_CAP_SYS_RESOURCE);
 168         } else {
 169                 cfs_cap_lower(CFS_CAP_SYS_RESOURCE);
 170         }
 171
 172         RETURN(0);
 173 }
 174
 175 #define GET_OA_ID(flag, oa) (flag == USRQUOTA ? oa->o_uid : oa->o_gid)
 176 static int filter_quota_getflag(struct obd_device *obd, struct obdo *oa)
 177 {
 178         struct obd_device_target *obt = &obd->u.obt;
 179         struct lustre_quota_ctxt *qctxt = &obt->obt_qctxt;
 180         int err, cnt, rc = 0;
 181         struct obd_quotactl *oqctl;
 182         ENTRY;
 183
 184         if (!ll_sb_any_quota_active(obt->obt_sb))
 185                 RETURN(0);
 186
 187         OBD_ALLOC_PTR(oqctl);
 188         if (!oqctl)
 189                 RETURN(-ENOMEM);
 190
 191         /* set over quota flags for a uid/gid */
 192         oa->o_valid |= OBD_MD_FLUSRQUOTA | OBD_MD_FLGRPQUOTA;
 193         oa->o_flags &= ~(OBD_FL_NO_USRQUOTA | OBD_FL_NO_GRPQUOTA);
 194
 195         for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
 196                 struct lustre_qunit_size *lqs = NULL;
 197
 198                 /* check if quota is enabled */
 199                 if (!ll_sb_has_quota_active(obt->obt_sb, cnt))
 200                         continue;
 201
 202                 lqs = quota_search_lqs(LQS_KEY(cnt, GET_OA_ID(cnt, oa)),
 203                                        qctxt, 0);
 204                 if (IS_ERR(lqs)) {
 205                         rc = PTR_ERR(lqs);
 206                         CDEBUG(D_QUOTA, "search lqs for %s %d failed, "
 207                                "(rc = %d)\n",
 208                                cnt == USRQUOTA ? "user" : "group",
 209                                GET_OA_ID(cnt, oa), rc);
 210                         break;
 211                 } else if (lqs == NULL) {
 212                         /* continue to check group quota if the file's owner
 213                          * doesn't have quota limit. LU-530 */
 214                         continue;
 215                 } else {
 216                         cfs_spin_lock(&lqs->lqs_lock);
 217                         if (lqs->lqs_bunit_sz <= qctxt->lqc_sync_blk) {
 218                                 oa->o_flags |= (cnt == USRQUOTA) ?
 219                                         OBD_FL_NO_USRQUOTA : OBD_FL_NO_GRPQUOTA;
 220                                 cfs_spin_unlock(&lqs->lqs_lock);
 221                                 CDEBUG(D_QUOTA, "set sync flag: bunit(%lu), "
 222                                        "sync_blk(%d)\n", lqs->lqs_bunit_sz,
 223                                        qctxt->lqc_sync_blk);
 224                                 /* this is for quota_search_lqs */
 225                                 lqs_putref(lqs);
 226                                 continue;
 227                         }
 228                         cfs_spin_unlock(&lqs->lqs_lock);
 229                         /* this is for quota_search_lqs */
 230                         lqs_putref(lqs);
 231                 }
 232
 233                 memset(oqctl, 0, sizeof(*oqctl));
 234
 235                 oqctl->qc_cmd = Q_GETQUOTA;
 236                 oqctl->qc_type = cnt;
 237                 oqctl->qc_id = (cnt == USRQUOTA) ? oa->o_uid : oa->o_gid;
 238                 err = fsfilt_quotactl(obd, obt->obt_sb, oqctl);
 239                 if (err) {
 240                         if (!rc)
 241                                 rc = err;
 242                         oa->o_valid &= ~((cnt == USRQUOTA) ? OBD_MD_FLUSRQUOTA :
 243                                                              OBD_MD_FLGRPQUOTA);
 244                         CDEBUG(D_QUOTA, "fsfilt getquota for %s %d failed, "
 245                                "(rc = %d)\n",
 246                                cnt == USRQUOTA ? "user" : "group",
 247                                cnt == USRQUOTA ? oa->o_uid : oa->o_gid, err);
 248                         continue;
 249                 }
 250
 251                 if (oqctl->qc_dqblk.dqb_bhardlimit &&
 252                    (toqb(oqctl->qc_dqblk.dqb_curspace) >=
 253                     oqctl->qc_dqblk.dqb_bhardlimit)) {
 254                         oa->o_flags |= (cnt == USRQUOTA) ?
 255                                 OBD_FL_NO_USRQUOTA : OBD_FL_NO_GRPQUOTA;
 256                         CDEBUG(D_QUOTA, "out of quota for %s %d\n",
 257                                cnt == USRQUOTA ? "user" : "group",
 258                                cnt == USRQUOTA ? oa->o_uid : oa->o_gid);
 259                 }
 260         }
 261         OBD_FREE_PTR(oqctl);
 262         RETURN(rc);
 263 }
 264
 265 /**
 266  * check whether the left quota of certain uid and gid can satisfy a block_write
 267  * or inode_create rpc. When need to acquire quota, return QUOTA_RET_ACQUOTA
 268  */
 269 static int quota_check_common(struct obd_device *obd, const unsigned int id[],
 270                               int pending[], int count, int cycle, int isblk,
 271                               struct inode *inode, int frags)
 272 {
 273         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 274         int i;
 275         struct qunit_data qdata[MAXQUOTAS];
 276         int mb = 0;
 277         int rc = 0, rc2[2] = { 0, 0 };
 278         ENTRY;
 279
 280         cfs_spin_lock(&qctxt->lqc_lock);
 281         if (!qctxt->lqc_valid){
 282                 cfs_spin_unlock(&qctxt->lqc_lock);
 283                 RETURN(rc);
 284         }
 285         cfs_spin_unlock(&qctxt->lqc_lock);
 286
 287         for (i = 0; i < MAXQUOTAS; i++) {
 288                 struct lustre_qunit_size *lqs = NULL;
 289
 290                 qdata[i].qd_id = id[i];
 291                 qdata[i].qd_flags = i;
 292                 if (isblk)
 293                         QDATA_SET_BLK(&qdata[i]);
 294                 qdata[i].qd_count = 0;
 295
 296                 /* check if quota is enabled */
 297                 if (!ll_sb_has_quota_active(qctxt->lqc_sb, i))
 298                         continue;
 299
 300                 /* ignore root user */
 301                 if (qdata[i].qd_id == 0 && !QDATA_IS_GRP(&qdata[i]))
 302                         continue;
 303
 304                 lqs = quota_search_lqs(LQS_KEY(i, id[i]), qctxt, 0);
 305                 if (lqs == NULL || IS_ERR(lqs))
 306                         continue;
 307
 308                 if (IS_ERR(lqs)) {
 309                         CERROR("can not find lqs for check_common: "
 310                                "[id %u] [%c] [isblk %d] [count %d] [rc %ld]\n",
 311                                id[i], i % 2 ? 'g': 'u', isblk, count,
 312                                PTR_ERR(lqs));
 313                         RETURN(PTR_ERR(lqs));
 314                 }
 315
 316                 rc2[i] = compute_remquota(obd, qctxt, &qdata[i], isblk);
 317                 cfs_spin_lock(&lqs->lqs_lock);
 318                 if (!cycle) {
 319                         if (isblk) {
 320                                 pending[i] = count * CFS_PAGE_SIZE;
 321                                 /* in order to complete this write, we need extra
 322                                  * meta blocks. This function can get it through
 323                                  * data needed to be written b=16542 */
 324                                 if (inode) {
 325                                         mb = pending[i];
 326                                         rc = fsfilt_get_mblk(obd, qctxt->lqc_sb,
 327                                                              &mb, inode,
 328                                                              frags);
 329                                         if (rc)
 330                                                 CERROR("%s: can't get extra "
 331                                                        "meta blocks\n",
 332                                                        obd->obd_name);
 333                                         else
 334                                                 pending[i] += mb;
 335                                 }
 336                                 LASSERTF(pending[i] >= 0, "pending is not valid"
 337                                          ", count=%d, mb=%d\n", count, mb);
 338                                 lqs->lqs_bwrite_pending += pending[i];
 339                         } else {
 340                                 pending[i] = count;
 341                                 lqs->lqs_iwrite_pending += pending[i];
 342                         }
 343                 }
 344
 345                 /* if xx_rec < 0, that means quota are releasing,
 346                  * and it may return before we use quota. So if
 347                  * we find this situation, we assuming it has
 348                  * returned b=18491 */
 349                 if (isblk && lqs->lqs_blk_rec < 0) {
 350                         if (qdata[i].qd_count < -lqs->lqs_blk_rec)
 351                                 qdata[i].qd_count = 0;
 352                         else
 353                                 qdata[i].qd_count += lqs->lqs_blk_rec;
 354                 }
 355                 if (!isblk && lqs->lqs_ino_rec < 0) {
 356                         if (qdata[i].qd_count < -lqs->lqs_ino_rec)
 357                                 qdata[i].qd_count = 0;
 358                         else
 359                                 qdata[i].qd_count += lqs->lqs_ino_rec;
 360                 }
 361
 362                 CDEBUG(D_QUOTA, "[id %u] [%c] [isblk %d] [count %d]"
 363                        " [lqs pending: %lu] [qd_count: "LPU64"] [metablocks: %d]"
 364                        " [pending: %d]\n", id[i], i % 2 ? 'g': 'u', isblk, count,
 365                        isblk ? lqs->lqs_bwrite_pending : lqs->lqs_iwrite_pending,
 366                        qdata[i].qd_count, mb, pending[i]);
 367                 if (rc2[i] == QUOTA_RET_OK) {
 368                         if (isblk && qdata[i].qd_count < lqs->lqs_bwrite_pending)
 369                                 rc2[i] = QUOTA_RET_ACQUOTA;
 370                         if (!isblk && qdata[i].qd_count <
 371                             lqs->lqs_iwrite_pending)
 372                                 rc2[i] = QUOTA_RET_ACQUOTA;
 373                 }
 374
 375                 cfs_spin_unlock(&lqs->lqs_lock);
 376
 377                 if (lqs->lqs_blk_rec  < 0 &&
 378                     qdata[i].qd_count <
 379                     lqs->lqs_bwrite_pending - lqs->lqs_blk_rec - mb)
 380                         OBD_FAIL_TIMEOUT(OBD_FAIL_QUOTA_DELAY_REL, 5);
 381
 382                 /* When cycle is zero, lqs_*_pending will be changed. We will
 383                  * get reference of the lqs here and put reference of lqs in
 384                  * quota_pending_commit b=14784 */
 385                 if (!cycle)
 386                         lqs_getref(lqs);
 387
 388                 /* this is for quota_search_lqs */
 389                 lqs_putref(lqs);
 390         }
 391
 392         if (rc2[0] == QUOTA_RET_ACQUOTA || rc2[1] == QUOTA_RET_ACQUOTA)
 393                 RETURN(QUOTA_RET_ACQUOTA);
 394         else
 395                 RETURN(rc);
 396 }
 397
 398 int quota_is_set(struct obd_device *obd, const unsigned int id[], int flag)
 399 {
 400         struct lustre_qunit_size *lqs;
 401         int i, q_set = 0;
 402
 403         if (!ll_sb_any_quota_active(obd->u.obt.obt_qctxt.lqc_sb))
 404                 RETURN(0);
 405
 406         for (i = 0; i < MAXQUOTAS; i++) {
 407                 /* check if quota is enabled */
 408                 if (!ll_sb_has_quota_active(obd->u.obt.obt_qctxt.lqc_sb, i))
 409                         continue;
 410                 lqs = quota_search_lqs(LQS_KEY(i, id[i]),
 411                                        &obd->u.obt.obt_qctxt, 0);
 412                 if (lqs && !IS_ERR(lqs)) {
 413                         if (lqs->lqs_flags & flag)
 414                                 q_set = 1;
 415                         lqs_putref(lqs);
 416                 }
 417         }
 418
 419         return q_set;
 420 }
 421
 422 static int quota_chk_acq_common(struct obd_device *obd, struct obd_export *exp,
 423                                 const unsigned int id[], int pending[],
 424                                 int count, quota_acquire acquire,
 425                                 struct obd_trans_info *oti, int isblk,
 426                                 struct inode *inode, int frags)
 427 {
 428         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 429         struct timeval work_start;
 430         struct timeval work_end;
 431         long timediff;
 432         struct l_wait_info lwi = { 0 };
 433         int rc = 0, cycle = 0, count_err = 1;
 434         ENTRY;
 435
 436         if (!quota_is_set(obd, id, isblk ? QB_SET : QI_SET))
 437                 RETURN(0);
 438
 439         if (isblk && (exp->exp_failed || exp->exp_abort_active_req))
 440                 /* If the client has been evicted or if it
 441                  * timed out and tried to reconnect already,
 442                  * abort the request immediately */
 443                 RETURN(-ENOTCONN);
 444
 445         CDEBUG(D_QUOTA, "check quota for %s\n", obd->obd_name);
 446         pending[USRQUOTA] = pending[GRPQUOTA] = 0;
 447         /* Unfortunately, if quota master is too busy to handle the
 448          * pre-dqacq in time and quota hash on ost is used up, we
 449          * have to wait for the completion of in flight dqacq/dqrel,
 450          * in order to get enough quota for write b=12588 */
 451         cfs_gettimeofday(&work_start);
 452         while ((rc = quota_check_common(obd, id, pending, count, cycle, isblk,
 453                                         inode, frags)) &
 454                QUOTA_RET_ACQUOTA) {
 455                 struct ptlrpc_thread *thr = oti != NULL ?
 456                                             oti->oti_thread : NULL;
 457
 458                 cfs_spin_lock(&qctxt->lqc_lock);
 459                 if (!qctxt->lqc_import && oti != NULL) {
 460                         cfs_spin_unlock(&qctxt->lqc_lock);
 461
 462                         LASSERT(thr != NULL);
 463                         /* The recovery thread doesn't have watchdog
 464                          * attached. LU-369 */
 465                         if (thr->t_watchdog != NULL)
 466                                 lc_watchdog_disable(thr->t_watchdog);
 467                         CDEBUG(D_QUOTA, "sleep for quota master\n");
 468                         l_wait_event(qctxt->lqc_wait_for_qmaster,
 469                                      check_qm(qctxt), &lwi);
 470
 471                         CDEBUG(D_QUOTA, "wake up when quota master is back\n");
 472                         if (thr->t_watchdog != NULL) {
 473                                 lc_watchdog_touch(thr->t_watchdog,
 474                                    ptlrpc_server_get_timeout(thr->t_svcpt));
 475                         }
 476                 } else {
 477                         cfs_spin_unlock(&qctxt->lqc_lock);
 478                 }
 479
 480                 cycle++;
 481                 if (isblk)
 482                         OBD_FAIL_TIMEOUT(OBD_FAIL_OST_HOLD_WRITE_RPC, 90);
 483                 /* after acquire(), we should run quota_check_common again
 484                  * so that we confirm there are enough quota to finish write */
 485                 rc = acquire(obd, id, oti, isblk);
 486
 487                 /* please reference to dqacq_completion for the below */
 488                 /* a new request is finished, try again */
 489                 if (rc == QUOTA_REQ_RETURNED) {
 490                         CDEBUG(D_QUOTA, "finish a quota req, try again\n");
 491                         continue;
 492                 }
 493
 494                 /* it is out of quota already */
 495                 if (rc == -EDQUOT) {
 496                         CDEBUG(D_QUOTA, "out of quota,  return -EDQUOT\n");
 497                         break;
 498                 }
 499
 500                 /* Related quota has been disabled by master, but enabled by
 501                  * slave, do not try again. */
 502                 if (unlikely(rc == -ESRCH)) {
 503                         CERROR("mismatched quota configuration, stop try.\n");
 504                         break;
 505                 }
 506
 507                 if (isblk && (exp->exp_failed || exp->exp_abort_active_req))
 508                         /* The client has been evicted or tried to
 509                          * to reconnect already, abort the request */
 510                         RETURN(-ENOTCONN);
 511
 512                 /* -EBUSY and others, wait a second and try again */
 513                 if (rc < 0) {
 514                         cfs_waitq_t        waitq;
 515                         struct l_wait_info lwi;
 516
 517                         if (thr != NULL && thr->t_watchdog != NULL)
 518                                 lc_watchdog_touch(thr->t_watchdog,
 519                                    ptlrpc_server_get_timeout(thr->t_svcpt));
 520                         CDEBUG(D_QUOTA, "rc: %d, count_err: %d\n", rc,
 521                                count_err++);
 522
 523                         cfs_waitq_init(&waitq);
 524                         lwi = LWI_TIMEOUT(cfs_time_seconds(min(cycle, 10)), NULL,
 525                                           NULL);
 526                         l_wait_event(waitq, 0, &lwi);
 527                 }
 528
 529                 if (rc < 0 || cycle % 10 == 0) {
 530                         cfs_spin_lock(&last_print_lock);
 531                         if (last_print == 0 ||
 532                             cfs_time_before((last_print + cfs_time_seconds(30)),
 533                                             cfs_time_current())) {
 534                                 last_print = cfs_time_current();
 535                                 cfs_spin_unlock(&last_print_lock);
 536                                 CWARN("still haven't managed to acquire quota "
 537                                       "space from the quota master after %d "
 538                                       "retries (err=%d, rc=%d)\n",
 539                                       cycle, count_err - 1, rc);
 540                         } else {
 541                                 cfs_spin_unlock(&last_print_lock);
 542                         }
 543                 }
 544
 545                 CDEBUG(D_QUOTA, "recheck quota with rc: %d, cycle: %d\n", rc,
 546                        cycle);
 547         }
 548         cfs_gettimeofday(&work_end);
 549         timediff = cfs_timeval_sub(&work_end, &work_start, NULL);
 550         lprocfs_counter_add(qctxt->lqc_stats,
 551                             isblk ? LQUOTA_WAIT_FOR_CHK_BLK :
 552                                     LQUOTA_WAIT_FOR_CHK_INO,
 553                             timediff);
 554
 555         if (rc > 0)
 556                 rc = 0;
 557         RETURN(rc);
 558 }
 559
 560 /**
 561  * when a block_write or inode_create rpc is finished, adjust the record for
 562  * pending blocks and inodes
 563  */
 564 static int quota_pending_commit(struct obd_device *obd, const unsigned int id[],
 565                                 int pending[], int isblk)
 566 {
 567         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 568         struct timeval work_start;
 569         struct timeval work_end;
 570         long timediff;
 571         int i;
 572         struct qunit_data qdata[MAXQUOTAS];
 573         ENTRY;
 574
 575         CDEBUG(D_QUOTA, "commit pending quota for  %s\n", obd->obd_name);
 576         CLASSERT(MAXQUOTAS < 4);
 577         if (!ll_sb_any_quota_active(qctxt->lqc_sb))
 578                 RETURN(0);
 579
 580         cfs_gettimeofday(&work_start);
 581         for (i = 0; i < MAXQUOTAS; i++) {
 582                 struct lustre_qunit_size *lqs = NULL;
 583
 584                 LASSERT(pending[i] >= 0);
 585                 if (pending[i] == 0)
 586                         continue;
 587
 588                 qdata[i].qd_id = id[i];
 589                 qdata[i].qd_flags = i;
 590                 if (isblk)
 591                         QDATA_SET_BLK(&qdata[i]);
 592                 qdata[i].qd_count = 0;
 593
 594                 if (qdata[i].qd_id == 0 && !QDATA_IS_GRP(&qdata[i]))
 595                         continue;
 596
 597                 lqs = quota_search_lqs(LQS_KEY(i, qdata[i].qd_id), qctxt, 0);
 598                 if (lqs == NULL || IS_ERR(lqs)) {
 599                         CERROR("can not find lqs for pending_commit: "
 600                                "[id %u] [%c] [pending %u] [isblk %d] (rc %ld), "
 601                                "maybe cause unexpected lqs refcount error!\n",
 602                                id[i], i ? 'g': 'u', pending[i], isblk,
 603                                lqs ? PTR_ERR(lqs) : -1);
 604                         continue;
 605                 }
 606
 607                 cfs_spin_lock(&lqs->lqs_lock);
 608                 if (isblk) {
 609                         LASSERTF(lqs->lqs_bwrite_pending >= pending[i],
 610                                  "there are too many blocks! [id %u] [%c] "
 611                                  "[bwrite_pending %lu] [pending %u]\n",
 612                                  id[i], i % 2 ? 'g' : 'u',
 613                                  lqs->lqs_bwrite_pending, pending[i]);
 614
 615                         lqs->lqs_bwrite_pending -= pending[i];
 616                 } else {
 617                         LASSERTF(lqs->lqs_iwrite_pending >= pending[i],
 618                                 "there are too many files! [id %u] [%c] "
 619                                 "[iwrite_pending %lu] [pending %u]\n",
 620                                 id[i], i % 2 ? 'g' : 'u',
 621                                 lqs->lqs_iwrite_pending, pending[i]);
 622
 623                         lqs->lqs_iwrite_pending -= pending[i];
 624                 }
 625                 CDEBUG(D_QUOTA, "%s: lqs_pending=%lu pending[%d]=%d isblk=%d\n",
 626                        obd->obd_name,
 627                        isblk ? lqs->lqs_bwrite_pending : lqs->lqs_iwrite_pending,
 628                        i, pending[i], isblk);
 629                 cfs_spin_unlock(&lqs->lqs_lock);
 630
 631                 /* for quota_search_lqs in pending_commit */
 632                 lqs_putref(lqs);
 633                 /* for quota_search_lqs in quota_check */
 634                 lqs_putref(lqs);
 635         }
 636         cfs_gettimeofday(&work_end);
 637         timediff = cfs_timeval_sub(&work_end, &work_start, NULL);
 638         lprocfs_counter_add(qctxt->lqc_stats,
 639                             isblk ? LQUOTA_WAIT_FOR_COMMIT_BLK :
 640                                     LQUOTA_WAIT_FOR_COMMIT_INO,
 641                             timediff);
 642
 643         RETURN(0);
 644 }
 645
 646 static int mds_quota_init(void)
 647 {
 648         return lustre_dquot_init();
 649 }
 650
 651 static int mds_quota_exit(void)
 652 {
 653         lustre_dquot_exit();
 654         return 0;
 655 }
 656
 657 static int mds_quota_setup(struct obd_device *obd)
 658 {
 659         struct obd_device_target *obt = &obd->u.obt;
 660         struct mds_obd *mds = &obd->u.mds;
 661         int rc;
 662         ENTRY;
 663
 664         if (unlikely(mds->mds_quota)) {
 665                 CWARN("try to reinitialize quota context!\n");
 666                 RETURN(0);
 667         }
 668
 669         cfs_init_rwsem(&obt->obt_rwsem);
 670         obt->obt_qfmt = LUSTRE_QUOTA_V2;
 671         mds->mds_quota_info.qi_version = LUSTRE_QUOTA_V2;
 672         cfs_sema_init(&obt->obt_quotachecking, 1);
 673         /* initialize quota master and quota context */
 674         cfs_init_rwsem(&mds->mds_qonoff_sem);
 675         rc = qctxt_init(obd, dqacq_handler);
 676         if (rc) {
 677                 CERROR("%s: initialize quota context failed! (rc:%d)\n",
 678                        obd->obd_name, rc);
 679                 RETURN(rc);
 680         }
 681         mds->mds_quota = 1;
 682         RETURN(rc);
 683 }
 684
 685 static int mds_quota_cleanup(struct obd_device *obd)
 686 {
 687         ENTRY;
 688         if (unlikely(!obd->u.mds.mds_quota))
 689                 RETURN(0);
 690
 691         qctxt_cleanup(&obd->u.obt.obt_qctxt, 0);
 692         RETURN(0);
 693 }
 694
 695 static int mds_quota_setinfo(struct obd_device *obd, void *data)
 696 {
 697         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 698         ENTRY;
 699
 700         if (unlikely(!obd->u.mds.mds_quota))
 701                 RETURN(0);
 702
 703         if (data != NULL)
 704                 QUOTA_MASTER_READY(qctxt);
 705         else
 706                 QUOTA_MASTER_UNREADY(qctxt);
 707         RETURN(0);
 708 }
 709
 710 static int mds_quota_fs_cleanup(struct obd_device *obd)
 711 {
 712         struct mds_obd *mds = &obd->u.mds;
 713         struct obd_quotactl oqctl;
 714         ENTRY;
 715
 716         if (unlikely(!mds->mds_quota))
 717                 RETURN(0);
 718
 719         mds->mds_quota = 0;
 720         memset(&oqctl, 0, sizeof(oqctl));
 721         oqctl.qc_type = UGQUOTA;
 722
 723         cfs_down_write(&mds->mds_qonoff_sem);
 724         mds_admin_quota_off(obd, &oqctl);
 725         cfs_up_write(&mds->mds_qonoff_sem);
 726         RETURN(0);
 727 }
 728
 729 static int quota_acquire_common(struct obd_device *obd, const unsigned int id[],
 730                                 struct obd_trans_info *oti, int isblk)
 731 {
 732         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 733         int rc;
 734         ENTRY;
 735
 736         rc = qctxt_adjust_qunit(obd, qctxt, id, isblk, 1, oti);
 737         RETURN(rc);
 738 }
 739
 740 quota_interface_t mds_quota_interface = {
 741         .quota_init     = mds_quota_init,
 742         .quota_exit     = mds_quota_exit,
 743         .quota_setup    = mds_quota_setup,
 744         .quota_cleanup  = mds_quota_cleanup,
 745         .quota_check    = target_quota_check,
 746         .quota_ctl      = mds_quota_ctl,
 747         .quota_setinfo  = mds_quota_setinfo,
 748         .quota_fs_cleanup = mds_quota_fs_cleanup,
 749         .quota_recovery = mds_quota_recovery,
 750         .quota_adjust   = mds_quota_adjust,
 751         .quota_chkquota = quota_chk_acq_common,
 752         .quota_acquire  = quota_acquire_common,
 753         .quota_pending_commit = quota_pending_commit,
 754 };
 755
 756 quota_interface_t filter_quota_interface = {
 757         .quota_setup    = filter_quota_setup,
 758         .quota_cleanup  = filter_quota_cleanup,
 759         .quota_check    = target_quota_check,
 760         .quota_ctl      = filter_quota_ctl,
 761         .quota_setinfo  = filter_quota_setinfo,
 762         .quota_clearinfo = filter_quota_clearinfo,
 763         .quota_enforce  = filter_quota_enforce,
 764         .quota_getflag  = filter_quota_getflag,
 765         .quota_acquire  = quota_acquire_common,
 766         .quota_adjust   = filter_quota_adjust,
 767         .quota_chkquota = quota_chk_acq_common,
 768         .quota_adjust_qunit   = filter_quota_adjust_qunit,
 769         .quota_pending_commit = quota_pending_commit,
 770 };
 771
 772 cfs_proc_dir_entry_t *lquota_type_proc_dir = NULL;
 773
 774 static int __init init_lustre_quota(void)
 775 {
 776         int rc = 0;
 777
 778         lquota_type_proc_dir = lprocfs_register(OBD_LQUOTA_DEVICENAME,
 779                                                 proc_lustre_root,
 780                                                 NULL, NULL);
 781         if (IS_ERR(lquota_type_proc_dir)) {
 782                 CERROR("LProcFS failed in lquota-init\n");
 783                 rc = PTR_ERR(lquota_type_proc_dir);
 784                 return rc;
 785         }
 786
 787         rc = qunit_cache_init();
 788         if (rc)
 789                 return rc;
 790
 791         PORTAL_SYMBOL_REGISTER(filter_quota_interface);
 792         PORTAL_SYMBOL_REGISTER(mds_quota_interface);
 793
 794         return 0;
 795 }
 796
 797 static void /*__exit*/ exit_lustre_quota(void)
 798 {
 799         PORTAL_SYMBOL_UNREGISTER(filter_quota_interface);
 800         PORTAL_SYMBOL_UNREGISTER(mds_quota_interface);
 801
 802         qunit_cache_cleanup();
 803
 804         if (lquota_type_proc_dir)
 805                 lprocfs_remove(&lquota_type_proc_dir);
 806 }
 807
 808 MODULE_AUTHOR("Sun Microsystems, Inc. <http://www.lustre.org/>");
 809 MODULE_DESCRIPTION("Lustre Quota");
 810 MODULE_LICENSE("GPL");
 811
 812 cfs_module(lquota, "1.0.0", init_lustre_quota, exit_lustre_quota);
 813
 814 EXPORT_SYMBOL(mds_quota_interface);
 815 EXPORT_SYMBOL(filter_quota_interface);
 816 #endif /* __KERNEL */