lustre/quota/quota_interface.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  */
  36
  37 #ifndef EXPORT_SYMTAB
  38 # define EXPORT_SYMTAB
  39 #endif
  40 #define DEBUG_SUBSYSTEM S_LQUOTA
  41
  42 #ifdef __KERNEL__
  43 # include <linux/version.h>
  44 # include <linux/module.h>
  45 # include <linux/init.h>
  46 # include <linux/fs.h>
  47 # include <linux/jbd.h>
  48 #  include <linux/smp_lock.h>
  49 #  include <linux/buffer_head.h>
  50 #  include <linux/workqueue.h>
  51 #  include <linux/mount.h>
  52 #else /* __KERNEL__ */
  53 # include <liblustre.h>
  54 #endif
  55
  56 #include <obd_class.h>
  57 #include <lustre_mds.h>
  58 #include <lustre_dlm.h>
  59 #include <lustre_cfg.h>
  60 #include <obd_ost.h>
  61 #include <lustre_fsfilt.h>
  62 #include <lustre_quota.h>
  63 #include <lprocfs_status.h>
  64 #include "quota_internal.h"
  65
  66 #ifdef __KERNEL__
  67
  68 #ifdef HAVE_QUOTA_SUPPORT
  69
  70 static cfs_time_t last_print = 0;
  71 static spinlock_t last_print_lock = SPIN_LOCK_UNLOCKED;
  72
  73 static int filter_quota_setup(struct obd_device *obd)
  74 {
  75         int rc = 0;
  76         struct obd_device_target *obt = &obd->u.obt;
  77         ENTRY;
  78
  79 #ifdef HAVE_QUOTA64
  80         obt->obt_qfmt = LUSTRE_QUOTA_V2;
  81 #else
  82         obt->obt_qfmt = LUSTRE_QUOTA_V1;
  83 #endif
  84         atomic_set(&obt->obt_quotachecking, 1);
  85         rc = qctxt_init(obd, NULL);
  86         if (rc)
  87                 CERROR("initialize quota context failed! (rc:%d)\n", rc);
  88
  89         RETURN(rc);
  90 }
  91
  92 static int filter_quota_cleanup(struct obd_device *obd)
  93 {
  94         qctxt_cleanup(&obd->u.obt.obt_qctxt, 0);
  95         return 0;
  96 }
  97
  98 static int filter_quota_setinfo(struct obd_export *exp, struct obd_device *obd)
  99 {
 100         struct obd_import *imp;
 101         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 102         ENTRY;
 103
 104         /* setup the quota context import */
 105         spin_lock(&obd->u.obt.obt_qctxt.lqc_lock);
 106         obd->u.obt.obt_qctxt.lqc_import = exp->exp_imp_reverse;
 107         spin_unlock(&obd->u.obt.obt_qctxt.lqc_lock);
 108         CDEBUG(D_QUOTA, "%s: lqc_import(%p) of obd(%p) is reactivated now, \n",
 109                obd->obd_name,exp->exp_imp_reverse, obd);
 110
 111         /* make imp's connect flags equal relative exp's connect flags
 112          * adding it to avoid the scan export list
 113          */
 114         imp = exp->exp_imp_reverse;
 115         if (imp)
 116                 imp->imp_connect_data.ocd_connect_flags |=
 117                         (exp->exp_connect_flags &
 118                          (OBD_CONNECT_QUOTA64 | OBD_CONNECT_CHANGE_QS));
 119
 120         cfs_waitq_signal(&qctxt->lqc_wait_for_qmaster);
 121         /* start quota slave recovery thread. (release high limits) */
 122         qslave_start_recovery(obd, &obd->u.obt.obt_qctxt);
 123         RETURN(0);
 124 }
 125
 126 static int filter_quota_clearinfo(struct obd_export *exp, struct obd_device *obd)
 127 {
 128         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 129         ENTRY;
 130
 131         /* lquota may be not set up before destroying export, b=14896 */
 132         if (!obd->obd_set_up)
 133                 RETURN(0);
 134
 135         /* when exp->exp_imp_reverse is destroyed, the corresponding lqc_import
 136          * should be invalid b=12374 */
 137         if (qctxt->lqc_import && qctxt->lqc_import == exp->exp_imp_reverse) {
 138                 spin_lock(&qctxt->lqc_lock);
 139                 qctxt->lqc_import = NULL;
 140                 spin_unlock(&qctxt->lqc_lock);
 141                 ptlrpc_cleanup_imp(exp->exp_imp_reverse);
 142                 dqacq_interrupt(qctxt);
 143                 CDEBUG(D_QUOTA, "%s: lqc_import of obd(%p) is invalid now.\n",
 144                        obd->obd_name, obd);
 145         }
 146         RETURN(0);
 147 }
 148
 149 static int filter_quota_enforce(struct obd_device *obd, unsigned int ignore)
 150 {
 151         ENTRY;
 152
 153         if (!sb_any_quota_enabled(obd->u.obt.obt_sb))
 154                 RETURN(0);
 155
 156         if (ignore) {
 157                 CDEBUG(D_QUOTA, "blocks will be written with ignoring quota.\n");
 158                 cfs_cap_raise(CFS_CAP_SYS_RESOURCE);
 159         } else {
 160                 cfs_cap_lower(CFS_CAP_SYS_RESOURCE);
 161         }
 162
 163         RETURN(0);
 164 }
 165
 166 #define GET_OA_ID(flag, oa) (flag == USRQUOTA ? oa->o_uid : oa->o_gid)
 167 static int filter_quota_getflag(struct obd_device *obd, struct obdo *oa)
 168 {
 169         struct obd_device_target *obt = &obd->u.obt;
 170         struct lustre_quota_ctxt *qctxt = &obt->obt_qctxt;
 171         int err, cnt, rc = 0;
 172         struct obd_quotactl *oqctl;
 173         ENTRY;
 174
 175         if (!sb_any_quota_enabled(obt->obt_sb))
 176                 RETURN(0);
 177
 178         OBD_ALLOC_PTR(oqctl);
 179         if (!oqctl) {
 180                 CERROR("Not enough memory!");
 181                 RETURN(-ENOMEM);
 182         }
 183
 184         /* set over quota flags for a uid/gid */
 185         oa->o_valid |= OBD_MD_FLUSRQUOTA | OBD_MD_FLGRPQUOTA;
 186         oa->o_flags &= ~(OBD_FL_NO_USRQUOTA | OBD_FL_NO_GRPQUOTA);
 187
 188         for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
 189                 struct lustre_qunit_size *lqs = NULL;
 190
 191                 lqs = quota_search_lqs(LQS_KEY(cnt, GET_OA_ID(cnt, oa)),
 192                                        qctxt, 0);
 193                 if (lqs == NULL || IS_ERR(lqs)) {
 194                         rc = PTR_ERR(lqs);
 195                         break;
 196                 } else {
 197                         spin_lock(&lqs->lqs_lock);
 198                         if (lqs->lqs_bunit_sz <= qctxt->lqc_sync_blk) {
 199                                 oa->o_flags |= (cnt == USRQUOTA) ?
 200                                         OBD_FL_NO_USRQUOTA : OBD_FL_NO_GRPQUOTA;
 201                                 CDEBUG(D_QUOTA, "set sync flag: bunit(%lu), "
 202                                        "sync_blk(%d)\n", lqs->lqs_bunit_sz,
 203                                        qctxt->lqc_sync_blk);
 204                                 spin_unlock(&lqs->lqs_lock);
 205                                 /* this is for quota_search_lqs */
 206                                 lqs_putref(lqs);
 207                                 continue;
 208                         }
 209                         spin_unlock(&lqs->lqs_lock);
 210                         /* this is for quota_search_lqs */
 211                         lqs_putref(lqs);
 212                 }
 213
 214                 memset(oqctl, 0, sizeof(*oqctl));
 215
 216                 oqctl->qc_cmd = Q_GETQUOTA;
 217                 oqctl->qc_type = cnt;
 218                 oqctl->qc_id = (cnt == USRQUOTA) ? oa->o_uid : oa->o_gid;
 219                 err = fsfilt_quotactl(obd, obt->obt_sb, oqctl);
 220                 if (err) {
 221                         if (!rc)
 222                                 rc = err;
 223                         oa->o_valid &= ~((cnt == USRQUOTA) ? OBD_MD_FLUSRQUOTA :
 224                                                              OBD_MD_FLGRPQUOTA);
 225                         continue;
 226                 }
 227
 228                 if (oqctl->qc_dqblk.dqb_bhardlimit &&
 229                    (toqb(oqctl->qc_dqblk.dqb_curspace) >=
 230                     oqctl->qc_dqblk.dqb_bhardlimit))
 231                         oa->o_flags |= (cnt == USRQUOTA) ?
 232                                 OBD_FL_NO_USRQUOTA : OBD_FL_NO_GRPQUOTA;
 233         }
 234         OBD_FREE_PTR(oqctl);
 235         RETURN(rc);
 236 }
 237
 238 static int filter_quota_acquire(struct obd_device *obd, unsigned int uid,
 239                                 unsigned int gid, struct obd_trans_info *oti)
 240 {
 241         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 242         int rc;
 243         ENTRY;
 244
 245         rc = qctxt_adjust_qunit(obd, qctxt, uid, gid, LQUOTA_FLAGS_BLK, 1, oti);
 246         RETURN(rc);
 247 }
 248
 249 /* check whether the left quota of certain uid and gid can satisfy a block_write
 250  * or inode_create rpc. When need to acquire quota, return QUOTA_RET_ACQUOTA */
 251 static int quota_check_common(struct obd_device *obd, unsigned int uid,
 252                               unsigned int gid, int count, int cycle, int isblk,
 253                               struct inode *inode, int frags, int *pending)
 254 {
 255         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 256         int i;
 257         __u32 id[MAXQUOTAS] = { uid, gid };
 258         struct qunit_data qdata[MAXQUOTAS];
 259         int mb = 0;
 260         int rc = 0, rc2[2] = { 0, 0 };
 261         ENTRY;
 262
 263         CLASSERT(MAXQUOTAS < 4);
 264         if (!sb_any_quota_enabled(qctxt->lqc_sb))
 265                 RETURN(rc);
 266
 267         spin_lock(&qctxt->lqc_lock);
 268         if (!qctxt->lqc_valid){
 269                 spin_unlock(&qctxt->lqc_lock);
 270                 RETURN(rc);
 271         }
 272         spin_unlock(&qctxt->lqc_lock);
 273
 274         for (i = 0; i < MAXQUOTAS; i++) {
 275                 struct lustre_qunit_size *lqs = NULL;
 276
 277                 qdata[i].qd_id = id[i];
 278                 qdata[i].qd_flags = i;
 279                 if (isblk)
 280                         QDATA_SET_BLK(&qdata[i]);
 281                 qdata[i].qd_count = 0;
 282
 283                 /* ignore root user */
 284                 if (qdata[i].qd_id == 0 && !QDATA_IS_GRP(&qdata[i]))
 285                         continue;
 286
 287                 lqs = quota_search_lqs(LQS_KEY(i, id[i]), qctxt, 0);
 288                 if (lqs == NULL || IS_ERR(lqs))
 289                         continue;
 290
 291                 rc2[i] = compute_remquota(obd, qctxt, &qdata[i], isblk);
 292                 spin_lock(&lqs->lqs_lock);
 293                 if (!cycle) {
 294                         if (isblk) {
 295                                 *pending = count * CFS_PAGE_SIZE;
 296                                 /* in order to complete this write, we need extra
 297                                  * meta blocks. This function can get it through
 298                                  * data needed to be written b=16542 */
 299                                 mb = *pending;
 300                                 LASSERT(inode && frags > 0);
 301                                 if (fsfilt_get_mblk(obd, qctxt->lqc_sb, &mb,
 302                                                     inode, frags) < 0)
 303                                         CDEBUG(D_ERROR,
 304                                                "can't get extra meta blocks.\n");
 305                                 else
 306                                         *pending += mb;
 307                                 lqs->lqs_bwrite_pending += *pending;
 308                         } else {
 309                                 *pending = count;
 310                                 lqs->lqs_iwrite_pending += *pending;
 311                         }
 312                 }
 313
 314                 /* if xx_rec < 0, that means quota are releasing,
 315                  * and it may return before we use quota. So if
 316                  * we find this situation, we assuming it has
 317                  * returned b=18491 */
 318                 if (isblk && lqs->lqs_blk_rec < 0) {
 319                         if (qdata[i].qd_count < -lqs->lqs_blk_rec)
 320                                 qdata[i].qd_count = 0;
 321                         else
 322                                 qdata[i].qd_count += lqs->lqs_blk_rec;
 323                 }
 324                 if (!isblk && lqs->lqs_ino_rec < 0) {
 325                         if (qdata[i].qd_count < -lqs->lqs_ino_rec)
 326                                 qdata[i].qd_count = 0;
 327                         else
 328                                 qdata[i].qd_count += lqs->lqs_ino_rec;
 329                 }
 330
 331                 CDEBUG(D_QUOTA, "count: %d, lqs pending: %lu, qd_count: "LPU64
 332                        ", metablocks: %d, isblk: %d, pending: %d.\n", count,
 333                        isblk ? lqs->lqs_bwrite_pending : lqs->lqs_iwrite_pending,
 334                        qdata[i].qd_count, mb, isblk, *pending);
 335                 if (rc2[i] == QUOTA_RET_OK) {
 336                         if (isblk && qdata[i].qd_count < lqs->lqs_bwrite_pending)
 337                                 rc2[i] = QUOTA_RET_ACQUOTA;
 338                         if (!isblk && qdata[i].qd_count <
 339                             lqs->lqs_iwrite_pending)
 340                                 rc2[i] = QUOTA_RET_ACQUOTA;
 341                 }
 342
 343                 spin_unlock(&lqs->lqs_lock);
 344
 345                 if (lqs->lqs_blk_rec  < 0 &&
 346                     qdata[i].qd_count <
 347                     lqs->lqs_bwrite_pending - lqs->lqs_blk_rec - mb)
 348                         OBD_FAIL_TIMEOUT(OBD_FAIL_QUOTA_DELAY_REL, 5);
 349
 350                 /* When cycle is zero, lqs_*_pending will be changed. We will
 351                  * get reference of the lqs here and put reference of lqs in
 352                  * quota_pending_commit b=14784 */
 353                 if (!cycle)
 354                         lqs_getref(lqs);
 355
 356                 /* this is for quota_search_lqs */
 357                 lqs_putref(lqs);
 358         }
 359
 360         if (rc2[0] == QUOTA_RET_ACQUOTA || rc2[1] == QUOTA_RET_ACQUOTA)
 361                 RETURN(QUOTA_RET_ACQUOTA);
 362         else
 363                 RETURN(rc);
 364 }
 365
 366 static int quota_chk_acq_common(struct obd_device *obd, unsigned int uid,
 367                                 unsigned int gid, int count, int *pending,
 368                                 int isblk, quota_acquire acquire,
 369                                 struct obd_trans_info *oti, struct inode *inode,
 370                                 int frags)
 371 {
 372         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 373         struct timeval work_start;
 374         struct timeval work_end;
 375         long timediff;
 376         struct l_wait_info lwi = { 0 };
 377         int rc = 0, cycle = 0, count_err = 1;
 378         ENTRY;
 379
 380         CDEBUG(D_QUOTA, "check quota for %s\n", obd->obd_name);
 381         /* Unfortunately, if quota master is too busy to handle the
 382          * pre-dqacq in time and quota hash on ost is used up, we
 383          * have to wait for the completion of in flight dqacq/dqrel,
 384          * in order to get enough quota for write b=12588 */
 385         do_gettimeofday(&work_start);
 386         while ((rc = quota_check_common(obd, uid, gid, count, cycle, isblk,
 387                                         inode, frags, pending)) & QUOTA_RET_ACQUOTA) {
 388
 389                 spin_lock(&qctxt->lqc_lock);
 390                 if (!qctxt->lqc_import && oti) {
 391                         spin_unlock(&qctxt->lqc_lock);
 392
 393                         LASSERT(oti && oti->oti_thread &&
 394                                 oti->oti_thread->t_watchdog);
 395
 396                         lc_watchdog_disable(oti->oti_thread->t_watchdog);
 397                         CDEBUG(D_QUOTA, "sleep for quota master\n");
 398                         l_wait_event(qctxt->lqc_wait_for_qmaster, check_qm(qctxt),
 399                                      &lwi);
 400                         CDEBUG(D_QUOTA, "wake up when quota master is back\n");
 401                         lc_watchdog_touch(oti->oti_thread->t_watchdog,
 402                                  GET_TIMEOUT(oti->oti_thread->t_svc));
 403                 } else {
 404                         spin_unlock(&qctxt->lqc_lock);
 405                 }
 406
 407                 cycle++;
 408                 if (isblk)
 409                         OBD_FAIL_TIMEOUT(OBD_FAIL_OST_HOLD_WRITE_RPC, 90);
 410                 /* after acquire(), we should run quota_check_common again
 411                  * so that we confirm there are enough quota to finish write */
 412                 rc = acquire(obd, uid, gid, oti);
 413
 414                 /* please reference to dqacq_completion for the below */
 415                 /* a new request is finished, try again */
 416                 if (rc == QUOTA_REQ_RETURNED) {
 417                         CDEBUG(D_QUOTA, "finish a quota req, try again\n");
 418                         continue;
 419                 }
 420
 421                 /* it is out of quota already */
 422                 if (rc == -EDQUOT) {
 423                         CDEBUG(D_QUOTA, "out of quota,  return -EDQUOT\n");
 424                         break;
 425                 }
 426
 427                 /* -EBUSY and others, wait a second and try again */
 428                 if (rc < 0) {
 429                         cfs_waitq_t        waitq;
 430                         struct l_wait_info lwi;
 431
 432                         if (oti && oti->oti_thread && oti->oti_thread->t_watchdog)
 433                                 lc_watchdog_touch(oti->oti_thread->t_watchdog,
 434                                          GET_TIMEOUT(oti->oti_thread->t_svc));
 435                         CDEBUG(D_QUOTA, "rc: %d, count_err: %d\n", rc,
 436                                count_err++);
 437
 438                         init_waitqueue_head(&waitq);
 439                         lwi = LWI_TIMEOUT(cfs_time_seconds(min(cycle, 10)), NULL,
 440                                           NULL);
 441                         l_wait_event(waitq, 0, &lwi);
 442                 }
 443
 444                 if (rc < 0 || cycle % 10 == 2) {
 445                         spin_lock(&last_print_lock);
 446                         if (last_print == 0 ||
 447                             cfs_time_before((last_print + cfs_time_seconds(30)),
 448                                             cfs_time_current())) {
 449                                 CWARN("still haven't managed to acquire quota "
 450                                       "space from the quota master after %d "
 451                                       "retries (err=%d, rc=%d)\n",
 452                                       cycle, count_err - 1, rc);
 453                                 last_print = cfs_time_current();
 454                         }
 455                         spin_unlock(&last_print_lock);
 456                 }
 457
 458                 CDEBUG(D_QUOTA, "recheck quota with rc: %d, cycle: %d\n", rc,
 459                        cycle);
 460         }
 461
 462         do_gettimeofday(&work_end);
 463         timediff = cfs_timeval_sub(&work_end, &work_start, NULL);
 464         lprocfs_counter_add(qctxt->lqc_stats,
 465                             isblk ? LQUOTA_WAIT_FOR_CHK_BLK :
 466                                     LQUOTA_WAIT_FOR_CHK_INO,
 467                             timediff);
 468
 469         RETURN(rc);
 470 }
 471
 472 static int filter_quota_check(struct obd_device *obd, unsigned int uid,
 473                               unsigned int gid, int npage, int *pending,
 474                               quota_acquire acquire, struct obd_trans_info *oti,
 475                               struct inode *inode, int frags)
 476 {
 477         return quota_chk_acq_common(obd, uid, gid, npage, pending, LQUOTA_FLAGS_BLK,
 478                                     acquire, oti, inode, frags);
 479 }
 480
 481 /* when a block_write or inode_create rpc is finished, adjust the record for
 482  * pending blocks and inodes*/
 483 static int quota_pending_commit(struct obd_device *obd, unsigned int uid,
 484                                 unsigned int gid, int pending, int isblk)
 485 {
 486         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 487         struct timeval work_start;
 488         struct timeval work_end;
 489         long timediff;
 490         int i;
 491         __u32 id[MAXQUOTAS] = { uid, gid };
 492         struct qunit_data qdata[MAXQUOTAS];
 493         ENTRY;
 494
 495         CDEBUG(D_QUOTA, "commit pending quota for  %s\n", obd->obd_name);
 496         CLASSERT(MAXQUOTAS < 4);
 497         if (!sb_any_quota_enabled(qctxt->lqc_sb))
 498                 RETURN(0);
 499
 500         do_gettimeofday(&work_start);
 501         for (i = 0; i < MAXQUOTAS; i++) {
 502                 struct lustre_qunit_size *lqs = NULL;
 503                 int flag = 0;
 504
 505                 qdata[i].qd_id = id[i];
 506                 qdata[i].qd_flags = i;
 507                 if (isblk)
 508                         QDATA_SET_BLK(&qdata[i]);
 509                 qdata[i].qd_count = 0;
 510
 511                 if (qdata[i].qd_id == 0 && !QDATA_IS_GRP(&qdata[i]))
 512                         continue;
 513
 514                 lqs = quota_search_lqs(LQS_KEY(i, qdata[i].qd_id), qctxt, 0);
 515                 if (lqs == NULL || IS_ERR(lqs))
 516                         continue;
 517
 518                 spin_lock(&lqs->lqs_lock);
 519                 if (isblk) {
 520                         if (lqs->lqs_bwrite_pending >= pending) {
 521                                 lqs->lqs_bwrite_pending -= pending;
 522                                 flag = 1;
 523                         } else {
 524                                 CDEBUG(D_ERROR, "there are too many blocks!\n");
 525                         }
 526                 } else {
 527                         if (lqs->lqs_iwrite_pending >= pending) {
 528                                 lqs->lqs_iwrite_pending -= pending;
 529                                 flag = 1;
 530                         } else {
 531                                 CDEBUG(D_ERROR, "there are too many files!\n");
 532                         }
 533                 }
 534                 CDEBUG(D_QUOTA, "lqs pending: %lu, pending: %d, isblk: %d.\n",
 535                        isblk ? lqs->lqs_bwrite_pending :
 536                        lqs->lqs_iwrite_pending, pending, isblk);
 537
 538                 spin_unlock(&lqs->lqs_lock);
 539                 lqs_putref(lqs);
 540                 /* When lqs_*_pening is changed back, we'll putref lqs
 541                  * here b=14784 */
 542                 if (flag)
 543                         lqs_putref(lqs);
 544         }
 545         do_gettimeofday(&work_end);
 546         timediff = cfs_timeval_sub(&work_end, &work_start, NULL);
 547         lprocfs_counter_add(qctxt->lqc_stats,
 548                             isblk ? LQUOTA_WAIT_FOR_COMMIT_BLK :
 549                                     LQUOTA_WAIT_FOR_COMMIT_INO,
 550                             timediff);
 551
 552         RETURN(0);
 553 }
 554
 555 static int filter_quota_pending_commit(struct obd_device *obd, unsigned int uid,
 556                                        unsigned int gid, int blocks)
 557 {
 558         return quota_pending_commit(obd, uid, gid, blocks, LQUOTA_FLAGS_BLK);
 559 }
 560
 561 static int mds_quota_init(void)
 562 {
 563         return lustre_dquot_init();
 564 }
 565
 566 static int mds_quota_exit(void)
 567 {
 568         lustre_dquot_exit();
 569         return 0;
 570 }
 571
 572 static int mds_quota_setup(struct obd_device *obd)
 573 {
 574         struct obd_device_target *obt = &obd->u.obt;
 575         struct mds_obd *mds = &obd->u.mds;
 576         int rc;
 577         ENTRY;
 578
 579 #ifdef HAVE_QUOTA64
 580         obt->obt_qfmt = LUSTRE_QUOTA_V2;
 581 #else
 582         obt->obt_qfmt = LUSTRE_QUOTA_V1;
 583 #endif
 584         mds->mds_quota_info.qi_version = LUSTRE_QUOTA_V2;
 585         atomic_set(&obt->obt_quotachecking, 1);
 586         /* initialize quota master and quota context */
 587         sema_init(&mds->mds_qonoff_sem, 1);
 588         rc = qctxt_init(obd, dqacq_handler);
 589         if (rc) {
 590                 CERROR("initialize quota context failed! (rc:%d)\n", rc);
 591                 RETURN(rc);
 592         }
 593         RETURN(rc);
 594 }
 595
 596 static int mds_quota_cleanup(struct obd_device *obd)
 597 {
 598         qctxt_cleanup(&obd->u.obt.obt_qctxt, 0);
 599         RETURN(0);
 600 }
 601
 602 static int mds_quota_fs_cleanup(struct obd_device *obd)
 603 {
 604         struct mds_obd *mds = &obd->u.mds;
 605         struct obd_quotactl oqctl;
 606         ENTRY;
 607
 608         memset(&oqctl, 0, sizeof(oqctl));
 609         oqctl.qc_type = UGQUOTA;
 610
 611         down(&mds->mds_qonoff_sem);
 612         mds_admin_quota_off(obd, &oqctl);
 613         up(&mds->mds_qonoff_sem);
 614         RETURN(0);
 615 }
 616
 617 static int mds_quota_check(struct obd_device *obd, unsigned int uid,
 618                            unsigned int gid, int inodes, int *pending,
 619                            quota_acquire acquire, struct obd_trans_info *oti,
 620                            struct inode *inode, int frags)
 621 {
 622         return quota_chk_acq_common(obd, uid, gid, inodes, pending, 0,
 623                                     acquire, oti, inode, frags);
 624 }
 625
 626 static int mds_quota_acquire(struct obd_device *obd, unsigned int uid,
 627                              unsigned int gid, struct obd_trans_info *oti)
 628 {
 629         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 630         int rc;
 631         ENTRY;
 632
 633         rc = qctxt_adjust_qunit(obd, qctxt, uid, gid, 0, 1, oti);
 634         RETURN(rc);
 635 }
 636
 637 static int mds_quota_pending_commit(struct obd_device *obd, unsigned int uid,
 638                                     unsigned int gid, int inodes)
 639 {
 640         return quota_pending_commit(obd, uid, gid, inodes, 0);
 641 }
 642 #endif /* HAVE_QUOTA_SUPPORT */
 643 #endif /* __KERNEL__ */
 644
 645 struct osc_quota_info {
 646         struct list_head        oqi_hash;       /* hash list */
 647         struct client_obd      *oqi_cli;        /* osc obd */
 648         unsigned int            oqi_id;         /* uid/gid of a file */
 649         short                   oqi_type;       /* quota type */
 650 };
 651
 652 spinlock_t qinfo_list_lock = SPIN_LOCK_UNLOCKED;
 653
 654 static struct list_head qinfo_hash[NR_DQHASH];
 655 /* SLAB cache for client quota context */
 656 cfs_mem_cache_t *qinfo_cachep = NULL;
 657
 658 static inline int hashfn(struct client_obd *cli, unsigned long id, int type)
 659                          __attribute__((__const__));
 660
 661 static inline int hashfn(struct client_obd *cli, unsigned long id, int type)
 662 {
 663         unsigned long tmp = ((unsigned long)cli>>6) ^ id;
 664         tmp = (tmp * (MAXQUOTAS - type)) % NR_DQHASH;
 665         return tmp;
 666 }
 667
 668 /* caller must hold qinfo_list_lock */
 669 static inline void insert_qinfo_hash(struct osc_quota_info *oqi)
 670 {
 671         struct list_head *head = qinfo_hash +
 672                 hashfn(oqi->oqi_cli, oqi->oqi_id, oqi->oqi_type);
 673
 674         LASSERT_SPIN_LOCKED(&qinfo_list_lock);
 675         list_add(&oqi->oqi_hash, head);
 676 }
 677
 678 /* caller must hold qinfo_list_lock */
 679 static inline void remove_qinfo_hash(struct osc_quota_info *oqi)
 680 {
 681         LASSERT_SPIN_LOCKED(&qinfo_list_lock);
 682         list_del_init(&oqi->oqi_hash);
 683 }
 684
 685 /* caller must hold qinfo_list_lock */
 686 static inline struct osc_quota_info *find_qinfo(struct client_obd *cli,
 687                                                 unsigned int id, int type)
 688 {
 689         unsigned int hashent = hashfn(cli, id, type);
 690         struct osc_quota_info *oqi;
 691
 692         LASSERT_SPIN_LOCKED(&qinfo_list_lock);
 693         list_for_each_entry(oqi, &qinfo_hash[hashent], oqi_hash) {
 694                 if (oqi->oqi_cli == cli &&
 695                     oqi->oqi_id == id && oqi->oqi_type == type)
 696                         return oqi;
 697         }
 698         return NULL;
 699 }
 700
 701 static struct osc_quota_info *alloc_qinfo(struct client_obd *cli,
 702                                           unsigned int id, int type)
 703 {
 704         struct osc_quota_info *oqi;
 705         ENTRY;
 706
 707         OBD_SLAB_ALLOC(oqi, qinfo_cachep, CFS_ALLOC_STD, sizeof(*oqi));
 708         if(!oqi)
 709                 RETURN(NULL);
 710
 711         CFS_INIT_LIST_HEAD(&oqi->oqi_hash);
 712         oqi->oqi_cli = cli;
 713         oqi->oqi_id = id;
 714         oqi->oqi_type = type;
 715
 716         RETURN(oqi);
 717 }
 718
 719 static void free_qinfo(struct osc_quota_info *oqi)
 720 {
 721         OBD_SLAB_FREE(oqi, qinfo_cachep, sizeof(*oqi));
 722 }
 723
 724 int osc_quota_chkdq(struct client_obd *cli, unsigned int uid, unsigned int gid)
 725 {
 726         unsigned int id;
 727         int cnt, rc = QUOTA_OK;
 728         ENTRY;
 729
 730         spin_lock(&qinfo_list_lock);
 731         for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
 732                 struct osc_quota_info *oqi = NULL;
 733
 734                 id = (cnt == USRQUOTA) ? uid : gid;
 735                 oqi = find_qinfo(cli, id, cnt);
 736                 if (oqi) {
 737                         rc = NO_QUOTA;
 738                         break;
 739                 }
 740         }
 741         spin_unlock(&qinfo_list_lock);
 742
 743         RETURN(rc);
 744 }
 745
 746 int osc_quota_setdq(struct client_obd *cli, unsigned int uid, unsigned int gid,
 747                     obd_flag valid, obd_flag flags)
 748 {
 749         unsigned int id;
 750         obd_flag noquota;
 751         int cnt, rc = 0;
 752         ENTRY;
 753
 754
 755         for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
 756                 struct osc_quota_info *oqi, *old;
 757
 758                 if (!(valid & ((cnt == USRQUOTA) ?
 759                     OBD_MD_FLUSRQUOTA : OBD_MD_FLGRPQUOTA)))
 760                         continue;
 761
 762                 id = (cnt == USRQUOTA) ? uid : gid;
 763                 noquota = (cnt == USRQUOTA) ?
 764                     (flags & OBD_FL_NO_USRQUOTA) : (flags & OBD_FL_NO_GRPQUOTA);
 765
 766                 oqi = alloc_qinfo(cli, id, cnt);
 767                 if (oqi) {
 768                         spin_lock(&qinfo_list_lock);
 769
 770                         old = find_qinfo(cli, id, cnt);
 771                         if (old && !noquota)
 772                                 remove_qinfo_hash(old);
 773                         else if (!old && noquota)
 774                                 insert_qinfo_hash(oqi);
 775
 776                         spin_unlock(&qinfo_list_lock);
 777
 778                         if (old || !noquota)
 779                                 free_qinfo(oqi);
 780                         if (old && !noquota)
 781                                 free_qinfo(old);
 782                 } else {
 783                         CERROR("not enough mem!\n");
 784                         rc = -ENOMEM;
 785                         break;
 786                 }
 787         }
 788
 789         RETURN(rc);
 790 }
 791
 792 int osc_quota_cleanup(struct obd_device *obd)
 793 {
 794         struct client_obd *cli = &obd->u.cli;
 795         struct osc_quota_info *oqi, *n;
 796         int i;
 797         ENTRY;
 798
 799         spin_lock(&qinfo_list_lock);
 800         for (i = 0; i < NR_DQHASH; i++) {
 801                 list_for_each_entry_safe(oqi, n, &qinfo_hash[i], oqi_hash) {
 802                         if (oqi->oqi_cli != cli)
 803                                 continue;
 804                         remove_qinfo_hash(oqi);
 805                         free_qinfo(oqi);
 806                 }
 807         }
 808         spin_unlock(&qinfo_list_lock);
 809
 810         RETURN(0);
 811 }
 812
 813 int osc_quota_init(void)
 814 {
 815         int i;
 816         ENTRY;
 817
 818         LASSERT(qinfo_cachep == NULL);
 819         qinfo_cachep = cfs_mem_cache_create("osc_quota_info",
 820                                             sizeof(struct osc_quota_info),
 821                                             0, 0);
 822         if (!qinfo_cachep)
 823                 RETURN(-ENOMEM);
 824
 825         for (i = 0; i < NR_DQHASH; i++)
 826                 CFS_INIT_LIST_HEAD(qinfo_hash + i);
 827
 828         RETURN(0);
 829 }
 830
 831 int osc_quota_exit(void)
 832 {
 833         struct osc_quota_info *oqi, *n;
 834         int i, rc;
 835         ENTRY;
 836
 837         spin_lock(&qinfo_list_lock);
 838         for (i = 0; i < NR_DQHASH; i++) {
 839                 list_for_each_entry_safe(oqi, n, &qinfo_hash[i], oqi_hash) {
 840                         remove_qinfo_hash(oqi);
 841                         free_qinfo(oqi);
 842                 }
 843         }
 844         spin_unlock(&qinfo_list_lock);
 845
 846         rc = cfs_mem_cache_destroy(qinfo_cachep);
 847         LASSERTF(rc == 0, "couldn't destory qinfo_cachep slab\n");
 848         qinfo_cachep = NULL;
 849
 850         RETURN(0);
 851 }
 852
 853 #ifdef __KERNEL__
 854 #ifdef HAVE_QUOTA_SUPPORT
 855 quota_interface_t mds_quota_interface = {
 856         .quota_init     = mds_quota_init,
 857         .quota_exit     = mds_quota_exit,
 858         .quota_setup    = mds_quota_setup,
 859         .quota_cleanup  = mds_quota_cleanup,
 860         .quota_check    = target_quota_check,
 861         .quota_ctl      = mds_quota_ctl,
 862         .quota_fs_cleanup       =mds_quota_fs_cleanup,
 863         .quota_recovery = mds_quota_recovery,
 864         .quota_adjust   = mds_quota_adjust,
 865         .quota_chkquota = mds_quota_check,
 866         .quota_acquire  = mds_quota_acquire,
 867         .quota_pending_commit = mds_quota_pending_commit,
 868 };
 869
 870 quota_interface_t filter_quota_interface = {
 871         .quota_setup    = filter_quota_setup,
 872         .quota_cleanup  = filter_quota_cleanup,
 873         .quota_check    = target_quota_check,
 874         .quota_ctl      = filter_quota_ctl,
 875         .quota_setinfo  = filter_quota_setinfo,
 876         .quota_clearinfo = filter_quota_clearinfo,
 877         .quota_enforce  = filter_quota_enforce,
 878         .quota_getflag  = filter_quota_getflag,
 879         .quota_acquire  = filter_quota_acquire,
 880         .quota_adjust   = filter_quota_adjust,
 881         .quota_chkquota = filter_quota_check,
 882         .quota_adjust_qunit   = filter_quota_adjust_qunit,
 883         .quota_pending_commit = filter_quota_pending_commit,
 884 };
 885 #endif
 886 #endif /* __KERNEL__ */
 887
 888 quota_interface_t mdc_quota_interface = {
 889         .quota_ctl      = client_quota_ctl,
 890         .quota_check    = client_quota_check,
 891         .quota_poll_check = client_quota_poll_check,
 892 };
 893
 894 quota_interface_t osc_quota_interface = {
 895         .quota_ctl      = client_quota_ctl,
 896         .quota_check    = client_quota_check,
 897         .quota_poll_check = client_quota_poll_check,
 898         .quota_init     = osc_quota_init,
 899         .quota_exit     = osc_quota_exit,
 900         .quota_chkdq    = osc_quota_chkdq,
 901         .quota_setdq    = osc_quota_setdq,
 902         .quota_cleanup  = osc_quota_cleanup,
 903         .quota_adjust_qunit = client_quota_adjust_qunit,
 904 };
 905
 906 quota_interface_t lov_quota_interface = {
 907         .quota_check    = lov_quota_check,
 908         .quota_ctl      = lov_quota_ctl,
 909         .quota_adjust_qunit = lov_quota_adjust_qunit,
 910 };
 911
 912 #ifdef __KERNEL__
 913
 914 cfs_proc_dir_entry_t *lquota_type_proc_dir = NULL;
 915
 916 static int __init init_lustre_quota(void)
 917 {
 918 #ifdef HAVE_QUOTA_SUPPORT
 919         int rc = 0;
 920
 921         lquota_type_proc_dir = lprocfs_register(OBD_LQUOTA_DEVICENAME,
 922                                                 proc_lustre_root,
 923                                                 NULL, NULL);
 924         if (IS_ERR(lquota_type_proc_dir)) {
 925                 CERROR("LProcFS failed in lquota-init\n");
 926                 rc = PTR_ERR(lquota_type_proc_dir);
 927                 return rc;
 928         }
 929
 930         rc = qunit_cache_init();
 931         if (rc)
 932                 return rc;
 933
 934         PORTAL_SYMBOL_REGISTER(filter_quota_interface);
 935         PORTAL_SYMBOL_REGISTER(mds_quota_interface);
 936 #endif
 937         PORTAL_SYMBOL_REGISTER(mdc_quota_interface);
 938         PORTAL_SYMBOL_REGISTER(osc_quota_interface);
 939         PORTAL_SYMBOL_REGISTER(lov_quota_interface);
 940         return 0;
 941 }
 942
 943 static void /*__exit*/ exit_lustre_quota(void)
 944 {
 945         PORTAL_SYMBOL_UNREGISTER(mdc_quota_interface);
 946         PORTAL_SYMBOL_UNREGISTER(osc_quota_interface);
 947         PORTAL_SYMBOL_UNREGISTER(lov_quota_interface);
 948 #ifdef HAVE_QUOTA_SUPPORT
 949         PORTAL_SYMBOL_UNREGISTER(filter_quota_interface);
 950         PORTAL_SYMBOL_UNREGISTER(mds_quota_interface);
 951
 952         qunit_cache_cleanup();
 953
 954         if (lquota_type_proc_dir)
 955                 lprocfs_remove(&lquota_type_proc_dir);
 956 #endif
 957 }
 958
 959 MODULE_AUTHOR("Sun Microsystems, Inc. <http://www.lustre.org/>");
 960 MODULE_DESCRIPTION("Lustre Quota");
 961 MODULE_LICENSE("GPL");
 962
 963 cfs_module(lquota, "1.0.0", init_lustre_quota, exit_lustre_quota);
 964
 965 #ifdef HAVE_QUOTA_SUPPORT
 966 EXPORT_SYMBOL(mds_quota_interface);
 967 EXPORT_SYMBOL(filter_quota_interface);
 968 #endif
 969 EXPORT_SYMBOL(mdc_quota_interface);
 970 EXPORT_SYMBOL(osc_quota_interface);
 971 EXPORT_SYMBOL(lov_quota_interface);
 972 #endif /* __KERNEL */