lustre/quota/quota_interface.c

   1 /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*-
   2  * vim:expandtab:shiftwidth=8:tabstop=8:
   3  *
   4  * GPL HEADER START
   5  *
   6  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   7  *
   8  * This program is free software; you can redistribute it and/or modify
   9  * it under the terms of the GNU General Public License version 2 only,
  10  * as published by the Free Software Foundation.
  11  *
  12  * This program is distributed in the hope that it will be useful, but
  13  * WITHOUT ANY WARRANTY; without even the implied warranty of
  14  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  15  * General Public License version 2 for more details (a copy is included
  16  * in the LICENSE file that accompanied this code).
  17  *
  18  * You should have received a copy of the GNU General Public License
  19  * version 2 along with this program; If not, see
  20  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  21  *
  22  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  23  * CA 95054 USA or visit www.sun.com if you need additional information or
  24  * have any questions.
  25  *
  26  * GPL HEADER END
  27  */
  28 /*
  29  * Copyright  2008 Sun Microsystems, Inc. All rights reserved
  30  * Use is subject to license terms.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  */
  36
  37 #ifndef EXPORT_SYMTAB
  38 # define EXPORT_SYMTAB
  39 #endif
  40 #define DEBUG_SUBSYSTEM S_LQUOTA
  41
  42 #ifdef __KERNEL__
  43 # include <linux/version.h>
  44 # include <linux/module.h>
  45 # include <linux/init.h>
  46 # include <linux/fs.h>
  47 #  include <linux/smp_lock.h>
  48 #  include <linux/buffer_head.h>
  49 #  include <linux/workqueue.h>
  50 #  include <linux/mount.h>
  51 #else /* __KERNEL__ */
  52 # include <liblustre.h>
  53 #endif
  54
  55 #include <obd_class.h>
  56 #include <lustre_mds.h>
  57 #include <lustre_dlm.h>
  58 #include <lustre_cfg.h>
  59 #include <obd_ost.h>
  60 #include <lustre_fsfilt.h>
  61 #include <lustre_quota.h>
  62 #include <lprocfs_status.h>
  63 #include "quota_internal.h"
  64
  65 #ifdef __KERNEL__
  66
  67 #ifdef HAVE_QUOTA_SUPPORT
  68
  69 static cfs_time_t last_print = 0;
  70 static spinlock_t last_print_lock = SPIN_LOCK_UNLOCKED;
  71
  72 static int filter_quota_setup(struct obd_device *obd)
  73 {
  74         int rc = 0;
  75         struct obd_device_target *obt = &obd->u.obt;
  76         ENTRY;
  77
  78 #ifdef HAVE_QUOTA64
  79         obt->obt_qfmt = LUSTRE_QUOTA_V2;
  80 #else
  81         obt->obt_qfmt = LUSTRE_QUOTA_V1;
  82 #endif
  83         atomic_set(&obt->obt_quotachecking, 1);
  84         rc = qctxt_init(obd, NULL);
  85         if (rc)
  86                 CERROR("initialize quota context failed! (rc:%d)\n", rc);
  87
  88         RETURN(rc);
  89 }
  90
  91 static int filter_quota_cleanup(struct obd_device *obd)
  92 {
  93         qctxt_cleanup(&obd->u.obt.obt_qctxt, 0);
  94         return 0;
  95 }
  96
  97 static int filter_quota_setinfo(struct obd_export *exp, struct obd_device *obd)
  98 {
  99         struct obd_import *imp;
 100         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 101         ENTRY;
 102
 103         /* setup the quota context import */
 104         spin_lock(&obd->u.obt.obt_qctxt.lqc_lock);
 105         obd->u.obt.obt_qctxt.lqc_import = exp->exp_imp_reverse;
 106         spin_unlock(&obd->u.obt.obt_qctxt.lqc_lock);
 107         CDEBUG(D_QUOTA, "%s: lqc_import(%p) of obd(%p) is reactivated now, \n",
 108                obd->obd_name,exp->exp_imp_reverse, obd);
 109
 110         /* make imp's connect flags equal relative exp's connect flags
 111          * adding it to avoid the scan export list
 112          */
 113         imp = exp->exp_imp_reverse;
 114         if (imp)
 115                 imp->imp_connect_data.ocd_connect_flags |=
 116                         (exp->exp_connect_flags &
 117                          (OBD_CONNECT_QUOTA64 | OBD_CONNECT_CHANGE_QS));
 118
 119         cfs_waitq_signal(&qctxt->lqc_wait_for_qmaster);
 120         /* start quota slave recovery thread. (release high limits) */
 121         qslave_start_recovery(obd, &obd->u.obt.obt_qctxt);
 122         RETURN(0);
 123 }
 124
 125 static int filter_quota_clearinfo(struct obd_export *exp, struct obd_device *obd)
 126 {
 127         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 128         ENTRY;
 129
 130         /* lquota may be not set up before destroying export, b=14896 */
 131         if (!obd->obd_set_up)
 132                 RETURN(0);
 133
 134         /* when exp->exp_imp_reverse is destroyed, the corresponding lqc_import
 135          * should be invalid b=12374 */
 136         if (qctxt->lqc_import && qctxt->lqc_import == exp->exp_imp_reverse) {
 137                 spin_lock(&qctxt->lqc_lock);
 138                 qctxt->lqc_import = NULL;
 139                 spin_unlock(&qctxt->lqc_lock);
 140                 ptlrpc_cleanup_imp(exp->exp_imp_reverse);
 141                 dqacq_interrupt(qctxt);
 142                 CDEBUG(D_QUOTA, "%s: lqc_import of obd(%p) is invalid now.\n",
 143                        obd->obd_name, obd);
 144         }
 145         RETURN(0);
 146 }
 147
 148 static int filter_quota_enforce(struct obd_device *obd, unsigned int ignore)
 149 {
 150         ENTRY;
 151
 152         if (!ll_sb_any_quota_active(obd->u.obt.obt_sb))
 153                 RETURN(0);
 154
 155         if (ignore) {
 156                 CDEBUG(D_QUOTA, "blocks will be written with ignoring quota.\n");
 157                 cfs_cap_raise(CFS_CAP_SYS_RESOURCE);
 158         } else {
 159                 cfs_cap_lower(CFS_CAP_SYS_RESOURCE);
 160         }
 161
 162         RETURN(0);
 163 }
 164
 165 #define GET_OA_ID(flag, oa) (flag == USRQUOTA ? oa->o_uid : oa->o_gid)
 166 static int filter_quota_getflag(struct obd_device *obd, struct obdo *oa)
 167 {
 168         struct obd_device_target *obt = &obd->u.obt;
 169         struct lustre_quota_ctxt *qctxt = &obt->obt_qctxt;
 170         int err, cnt, rc = 0;
 171         struct obd_quotactl *oqctl;
 172         ENTRY;
 173
 174         if (!ll_sb_any_quota_active(obt->obt_sb))
 175                 RETURN(0);
 176
 177         OBD_ALLOC_PTR(oqctl);
 178         if (!oqctl)
 179                 RETURN(-ENOMEM);
 180
 181         /* set over quota flags for a uid/gid */
 182         oa->o_valid |= OBD_MD_FLUSRQUOTA | OBD_MD_FLGRPQUOTA;
 183         oa->o_flags &= ~(OBD_FL_NO_USRQUOTA | OBD_FL_NO_GRPQUOTA);
 184
 185         for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
 186                 struct lustre_qunit_size *lqs = NULL;
 187
 188                 lqs = quota_search_lqs(LQS_KEY(cnt, GET_OA_ID(cnt, oa)),
 189                                        qctxt, 0);
 190                 if (lqs == NULL || IS_ERR(lqs)) {
 191                         rc = PTR_ERR(lqs);
 192                         break;
 193                 } else {
 194                         spin_lock(&lqs->lqs_lock);
 195                         if (lqs->lqs_bunit_sz <= qctxt->lqc_sync_blk) {
 196                                 oa->o_flags |= (cnt == USRQUOTA) ?
 197                                         OBD_FL_NO_USRQUOTA : OBD_FL_NO_GRPQUOTA;
 198                                 CDEBUG(D_QUOTA, "set sync flag: bunit(%lu), "
 199                                        "sync_blk(%d)\n", lqs->lqs_bunit_sz,
 200                                        qctxt->lqc_sync_blk);
 201                                 spin_unlock(&lqs->lqs_lock);
 202                                 /* this is for quota_search_lqs */
 203                                 lqs_putref(lqs);
 204                                 continue;
 205                         }
 206                         spin_unlock(&lqs->lqs_lock);
 207                         /* this is for quota_search_lqs */
 208                         lqs_putref(lqs);
 209                 }
 210
 211                 memset(oqctl, 0, sizeof(*oqctl));
 212
 213                 oqctl->qc_cmd = Q_GETQUOTA;
 214                 oqctl->qc_type = cnt;
 215                 oqctl->qc_id = (cnt == USRQUOTA) ? oa->o_uid : oa->o_gid;
 216                 err = fsfilt_quotactl(obd, obt->obt_sb, oqctl);
 217                 if (err) {
 218                         if (!rc)
 219                                 rc = err;
 220                         oa->o_valid &= ~((cnt == USRQUOTA) ? OBD_MD_FLUSRQUOTA :
 221                                                              OBD_MD_FLGRPQUOTA);
 222                         continue;
 223                 }
 224
 225                 if (oqctl->qc_dqblk.dqb_bhardlimit &&
 226                    (toqb(oqctl->qc_dqblk.dqb_curspace) >=
 227                     oqctl->qc_dqblk.dqb_bhardlimit))
 228                         oa->o_flags |= (cnt == USRQUOTA) ?
 229                                 OBD_FL_NO_USRQUOTA : OBD_FL_NO_GRPQUOTA;
 230         }
 231         OBD_FREE_PTR(oqctl);
 232         RETURN(rc);
 233 }
 234
 235 static int filter_quota_acquire(struct obd_device *obd, unsigned int uid,
 236                                 unsigned int gid, struct obd_trans_info *oti)
 237 {
 238         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 239         int rc;
 240         ENTRY;
 241
 242         rc = qctxt_adjust_qunit(obd, qctxt, uid, gid, LQUOTA_FLAGS_BLK, 1, oti);
 243         RETURN(rc);
 244 }
 245
 246 /* check whether the left quota of certain uid and gid can satisfy a block_write
 247  * or inode_create rpc. When need to acquire quota, return QUOTA_RET_ACQUOTA */
 248 static int quota_check_common(struct obd_device *obd, unsigned int uid,
 249                               unsigned int gid, int count, int cycle, int isblk,
 250                               struct inode *inode, int frags, int pending[2])
 251 {
 252         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 253         int i;
 254         __u32 id[MAXQUOTAS] = { uid, gid };
 255         struct qunit_data qdata[MAXQUOTAS];
 256         int mb = 0;
 257         int rc = 0, rc2[2] = { 0, 0 };
 258         ENTRY;
 259
 260         spin_lock(&qctxt->lqc_lock);
 261         if (!qctxt->lqc_valid){
 262                 spin_unlock(&qctxt->lqc_lock);
 263                 RETURN(rc);
 264         }
 265         spin_unlock(&qctxt->lqc_lock);
 266
 267         for (i = 0; i < MAXQUOTAS; i++) {
 268                 struct lustre_qunit_size *lqs = NULL;
 269
 270                 qdata[i].qd_id = id[i];
 271                 qdata[i].qd_flags = i;
 272                 if (isblk)
 273                         QDATA_SET_BLK(&qdata[i]);
 274                 qdata[i].qd_count = 0;
 275
 276                 /* ignore root user */
 277                 if (qdata[i].qd_id == 0 && !QDATA_IS_GRP(&qdata[i]))
 278                         continue;
 279
 280                 lqs = quota_search_lqs(LQS_KEY(i, id[i]), qctxt, 0);
 281                 if (lqs == NULL || IS_ERR(lqs))
 282                         continue;
 283
 284                 rc2[i] = compute_remquota(obd, qctxt, &qdata[i], isblk);
 285                 spin_lock(&lqs->lqs_lock);
 286                 if (!cycle) {
 287                         if (isblk) {
 288                                 pending[i] = count * CFS_PAGE_SIZE;
 289                                 /* in order to complete this write, we need extra
 290                                  * meta blocks. This function can get it through
 291                                  * data needed to be written b=16542 */
 292                                 mb = pending[i];
 293                                 LASSERT(inode && frags > 0);
 294                                 if (fsfilt_get_mblk(obd, qctxt->lqc_sb, &mb,
 295                                                     inode, frags) < 0)
 296                                         CERROR("%s: can't get extra meta "
 297                                                "blocks\n", obd->obd_name);
 298                                 else
 299                                         pending[i] += mb;
 300                                 lqs->lqs_bwrite_pending += pending[i];
 301                         } else {
 302                                 pending[i] = count;
 303                                 lqs->lqs_iwrite_pending += pending[i];
 304                         }
 305                 }
 306
 307                 /* if xx_rec < 0, that means quota are releasing,
 308                  * and it may return before we use quota. So if
 309                  * we find this situation, we assuming it has
 310                  * returned b=18491 */
 311                 if (isblk && lqs->lqs_blk_rec < 0) {
 312                         if (qdata[i].qd_count < -lqs->lqs_blk_rec)
 313                                 qdata[i].qd_count = 0;
 314                         else
 315                                 qdata[i].qd_count += lqs->lqs_blk_rec;
 316                 }
 317                 if (!isblk && lqs->lqs_ino_rec < 0) {
 318                         if (qdata[i].qd_count < -lqs->lqs_ino_rec)
 319                                 qdata[i].qd_count = 0;
 320                         else
 321                                 qdata[i].qd_count += lqs->lqs_ino_rec;
 322                 }
 323
 324                 CDEBUG(D_QUOTA, "count=%d lqs_pending=%lu qd_count="LPU64
 325                        " isblk=%d mb=%d pending[%d]=%d\n", count,
 326                        isblk ? lqs->lqs_bwrite_pending : lqs->lqs_iwrite_pending,
 327                        qdata[i].qd_count, isblk, mb, i, pending[i]);
 328                 if (rc2[i] == QUOTA_RET_OK) {
 329                         if (isblk && qdata[i].qd_count < lqs->lqs_bwrite_pending)
 330                                 rc2[i] = QUOTA_RET_ACQUOTA;
 331                         if (!isblk && qdata[i].qd_count <
 332                             lqs->lqs_iwrite_pending)
 333                                 rc2[i] = QUOTA_RET_ACQUOTA;
 334                 }
 335
 336                 spin_unlock(&lqs->lqs_lock);
 337
 338                 if (lqs->lqs_blk_rec  < 0 &&
 339                     qdata[i].qd_count <
 340                     lqs->lqs_bwrite_pending - lqs->lqs_blk_rec - mb)
 341                         OBD_FAIL_TIMEOUT(OBD_FAIL_QUOTA_DELAY_REL, 5);
 342
 343                 /* When cycle is zero, lqs_*_pending will be changed. We will
 344                  * get reference of the lqs here and put reference of lqs in
 345                  * quota_pending_commit b=14784 */
 346                 if (!cycle)
 347                         lqs_getref(lqs);
 348
 349                 /* this is for quota_search_lqs */
 350                 lqs_putref(lqs);
 351         }
 352
 353         if (rc2[0] == QUOTA_RET_ACQUOTA || rc2[1] == QUOTA_RET_ACQUOTA)
 354                 RETURN(QUOTA_RET_ACQUOTA);
 355         else
 356                 RETURN(rc);
 357 }
 358
 359 static int quota_chk_acq_common(struct obd_export *exp, unsigned int uid,
 360                                 unsigned int gid, int count, int pending[2],
 361                                 int isblk, quota_acquire acquire,
 362                                 struct obd_trans_info *oti, struct inode *inode,
 363                                 int frags)
 364 {
 365         struct obd_device *obd = exp->exp_obd;
 366         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 367         struct timeval work_start;
 368         struct timeval work_end;
 369         long timediff;
 370         struct l_wait_info lwi = { 0 };
 371         int rc = 0, cycle = 0, count_err = 1;
 372         ENTRY;
 373
 374         CDEBUG(D_QUOTA, "check quota for %s\n", obd->obd_name);
 375         if (isblk && (exp->exp_failed || exp->exp_abort_active_req))
 376                 /* If the client has been evicted or if it
 377                  * timed out and tried to reconnect already,
 378                  * abort the request immediately */
 379                 RETURN(-ENOTCONN);
 380
 381         /* Unfortunately, if quota master is too busy to handle the
 382          * pre-dqacq in time and quota hash on ost is used up, we
 383          * have to wait for the completion of in flight dqacq/dqrel,
 384          * in order to get enough quota for write b=12588 */
 385         do_gettimeofday(&work_start);
 386         while ((rc = quota_check_common(obd, uid, gid, count, cycle, isblk,
 387                                         inode, frags, pending)) & QUOTA_RET_ACQUOTA) {
 388
 389                 spin_lock(&qctxt->lqc_lock);
 390                 if (!qctxt->lqc_import && oti) {
 391                         spin_unlock(&qctxt->lqc_lock);
 392
 393                         LASSERT(oti && oti->oti_thread &&
 394                                 oti->oti_thread->t_watchdog);
 395
 396                         lc_watchdog_disable(oti->oti_thread->t_watchdog);
 397                         CDEBUG(D_QUOTA, "sleep for quota master\n");
 398                         l_wait_event(qctxt->lqc_wait_for_qmaster, check_qm(qctxt),
 399                                      &lwi);
 400                         CDEBUG(D_QUOTA, "wake up when quota master is back\n");
 401                         lc_watchdog_touch(oti->oti_thread->t_watchdog,
 402                                  GET_TIMEOUT(oti->oti_thread->t_svc));
 403                 } else {
 404                         spin_unlock(&qctxt->lqc_lock);
 405                 }
 406
 407                 cycle++;
 408                 if (isblk)
 409                         OBD_FAIL_TIMEOUT(OBD_FAIL_OST_HOLD_WRITE_RPC, 90);
 410                 /* after acquire(), we should run quota_check_common again
 411                  * so that we confirm there are enough quota to finish write */
 412                 rc = acquire(obd, uid, gid, oti);
 413
 414                 /* please reference to dqacq_completion for the below */
 415                 /* a new request is finished, try again */
 416                 if (rc == QUOTA_REQ_RETURNED) {
 417                         CDEBUG(D_QUOTA, "finish a quota req, try again\n");
 418                         continue;
 419                 }
 420
 421                 /* it is out of quota already */
 422                 if (rc == -EDQUOT) {
 423                         CDEBUG(D_QUOTA, "out of quota,  return -EDQUOT\n");
 424                         break;
 425                 }
 426
 427                 if (isblk && (exp->exp_failed || exp->exp_abort_active_req))
 428                         /* The client has been evicted or tried to
 429                          * to reconnect already, abort the request */
 430                         RETURN(-ENOTCONN);
 431
 432                 /* -EBUSY and others, wait a second and try again */
 433                 if (rc < 0) {
 434                         cfs_waitq_t        waitq;
 435                         struct l_wait_info lwi;
 436
 437                         if (oti && oti->oti_thread && oti->oti_thread->t_watchdog)
 438                                 lc_watchdog_touch(oti->oti_thread->t_watchdog,
 439                                          GET_TIMEOUT(oti->oti_thread->t_svc));
 440                         CDEBUG(D_QUOTA, "rc: %d, count_err: %d\n", rc,
 441                                count_err++);
 442
 443                         init_waitqueue_head(&waitq);
 444                         lwi = LWI_TIMEOUT(cfs_time_seconds(min(cycle, 10)), NULL,
 445                                           NULL);
 446                         l_wait_event(waitq, 0, &lwi);
 447                 }
 448
 449                 if (rc < 0 || cycle % 10 == 0) {
 450                         spin_lock(&last_print_lock);
 451                         if (last_print == 0 ||
 452                             cfs_time_before((last_print + cfs_time_seconds(30)),
 453                                             cfs_time_current())) {
 454                                 CWARN("still haven't managed to acquire quota "
 455                                       "space from the quota master after %d "
 456                                       "retries (err=%d, rc=%d)\n",
 457                                       cycle, count_err - 1, rc);
 458                                 last_print = cfs_time_current();
 459                         }
 460                         spin_unlock(&last_print_lock);
 461                 }
 462
 463                 CDEBUG(D_QUOTA, "recheck quota with rc: %d, cycle: %d\n", rc,
 464                        cycle);
 465         }
 466
 467         do_gettimeofday(&work_end);
 468         timediff = cfs_timeval_sub(&work_end, &work_start, NULL);
 469         lprocfs_counter_add(qctxt->lqc_stats,
 470                             isblk ? LQUOTA_WAIT_FOR_CHK_BLK :
 471                                     LQUOTA_WAIT_FOR_CHK_INO,
 472                             timediff);
 473
 474         if (rc > 0)
 475                 rc = 0;
 476         RETURN(rc);
 477 }
 478
 479 int quota_is_set(struct obd_device *obd, unsigned int uid,
 480                  unsigned int gid, int flag)
 481 {
 482         struct lustre_qunit_size *lqs;
 483         __u32 id[MAXQUOTAS] = { uid, gid };
 484         int i, q_set = 0;
 485
 486         if (!ll_sb_any_quota_active(obd->u.obt.obt_qctxt.lqc_sb))
 487                 RETURN(0);
 488
 489         for (i = 0; i < MAXQUOTAS; i++) {
 490                 lqs = quota_search_lqs(LQS_KEY(i, id[i]),
 491                                        &obd->u.obt.obt_qctxt, 0);
 492                 if (lqs && !IS_ERR(lqs)) {
 493                         if (lqs->lqs_flags & flag)
 494                                 q_set = 1;
 495                         lqs_putref(lqs);
 496                 }
 497         }
 498
 499         return q_set;
 500 }
 501
 502 static int filter_quota_check(struct obd_export *exp, unsigned int uid,
 503                               unsigned int gid, int npage, int pending[2],
 504                               quota_acquire acquire, struct obd_trans_info *oti,
 505                               struct inode *inode, int frags)
 506 {
 507         return quota_is_set(exp->exp_obd, uid, gid, QB_SET) ?
 508                 quota_chk_acq_common(exp, uid, gid, npage, pending,
 509                                      LQUOTA_FLAGS_BLK, acquire, oti, inode,
 510                                      frags) : 0;
 511 }
 512
 513 /* when a block_write or inode_create rpc is finished, adjust the record for
 514  * pending blocks and inodes*/
 515 static int quota_pending_commit(struct obd_device *obd, unsigned int uid,
 516                                 unsigned int gid, int pending[2], int isblk)
 517 {
 518         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 519         struct timeval work_start;
 520         struct timeval work_end;
 521         long timediff;
 522         int i;
 523         __u32 id[MAXQUOTAS] = { uid, gid };
 524         struct qunit_data qdata[MAXQUOTAS];
 525         ENTRY;
 526
 527         CDEBUG(D_QUOTA, "%s: commit pending quota\n", obd->obd_name);
 528         CLASSERT(MAXQUOTAS < 4);
 529
 530         do_gettimeofday(&work_start);
 531         for (i = 0; i < MAXQUOTAS; i++) {
 532                 struct lustre_qunit_size *lqs = NULL;
 533                 int flag = 0;
 534
 535                 qdata[i].qd_id = id[i];
 536                 qdata[i].qd_flags = i;
 537                 if (isblk)
 538                         QDATA_SET_BLK(&qdata[i]);
 539                 qdata[i].qd_count = 0;
 540
 541                 if (qdata[i].qd_id == 0 && !QDATA_IS_GRP(&qdata[i]))
 542                         continue;
 543
 544                 lqs = quota_search_lqs(LQS_KEY(i, qdata[i].qd_id), qctxt, 0);
 545                 if (lqs == NULL || IS_ERR(lqs))
 546                         continue;
 547
 548                 spin_lock(&lqs->lqs_lock);
 549                 if (isblk) {
 550                         if (lqs->lqs_bwrite_pending >= pending[i]) {
 551                                 lqs->lqs_bwrite_pending -= pending[i];
 552                                 flag = 1;
 553                         } else {
 554                                 CERROR("%s: there are too many blocks!\n",
 555                                        obd->obd_name);
 556                         }
 557                 } else {
 558                         if (lqs->lqs_iwrite_pending >= pending[i]) {
 559                                 lqs->lqs_iwrite_pending -= pending[i];
 560                                 flag = 1;
 561                         } else {
 562                                 CERROR("%s: there are too many files!\n",
 563                                        obd->obd_name);
 564                         }
 565                 }
 566                 CDEBUG(D_QUOTA, "%s: lqs_pending=%lu pending[%d]=%d isblk=%d\n",
 567                        obd->obd_name,
 568                        isblk ? lqs->lqs_bwrite_pending : lqs->lqs_iwrite_pending,
 569                        i, pending[i], isblk);
 570
 571                 spin_unlock(&lqs->lqs_lock);
 572                 lqs_putref(lqs);
 573                 /* When lqs_*_pening is changed back, we'll putref lqs
 574                  * here b=14784 */
 575                 if (flag)
 576                         lqs_putref(lqs);
 577         }
 578         do_gettimeofday(&work_end);
 579         timediff = cfs_timeval_sub(&work_end, &work_start, NULL);
 580         lprocfs_counter_add(qctxt->lqc_stats,
 581                             isblk ? LQUOTA_WAIT_FOR_COMMIT_BLK :
 582                                     LQUOTA_WAIT_FOR_COMMIT_INO,
 583                             timediff);
 584
 585         RETURN(0);
 586 }
 587
 588 static int filter_quota_pending_commit(struct obd_device *obd, unsigned int uid,
 589                                        unsigned int gid, int pending[2])
 590 {
 591         return quota_pending_commit(obd, uid, gid, pending, LQUOTA_FLAGS_BLK);
 592 }
 593
 594 static int mds_quota_init(void)
 595 {
 596         return lustre_dquot_init();
 597 }
 598
 599 static int mds_quota_exit(void)
 600 {
 601         lustre_dquot_exit();
 602         return 0;
 603 }
 604
 605 static int mds_quota_setup(struct obd_device *obd)
 606 {
 607         struct obd_device_target *obt = &obd->u.obt;
 608         struct mds_obd *mds = &obd->u.mds;
 609         int rc;
 610         ENTRY;
 611
 612 #ifdef HAVE_QUOTA64
 613         obt->obt_qfmt = LUSTRE_QUOTA_V2;
 614 #else
 615         obt->obt_qfmt = LUSTRE_QUOTA_V1;
 616 #endif
 617         mds->mds_quota_info.qi_version = LUSTRE_QUOTA_V2;
 618         atomic_set(&obt->obt_quotachecking, 1);
 619         /* initialize quota master and quota context */
 620         sema_init(&mds->mds_qonoff_sem, 1);
 621         rc = qctxt_init(obd, dqacq_handler);
 622         if (rc) {
 623                 CERROR("%s: initialize quota context failed! (rc:%d)\n",
 624                        obd->obd_name, rc);
 625                 RETURN(rc);
 626         }
 627         RETURN(rc);
 628 }
 629
 630 static int mds_quota_cleanup(struct obd_device *obd)
 631 {
 632         qctxt_cleanup(&obd->u.obt.obt_qctxt, 0);
 633         RETURN(0);
 634 }
 635
 636 static int mds_quota_fs_cleanup(struct obd_device *obd)
 637 {
 638         struct mds_obd *mds = &obd->u.mds;
 639         struct obd_quotactl oqctl;
 640         ENTRY;
 641
 642         memset(&oqctl, 0, sizeof(oqctl));
 643         oqctl.qc_type = UGQUOTA;
 644
 645         down(&mds->mds_qonoff_sem);
 646         mds_admin_quota_off(obd, &oqctl);
 647         up(&mds->mds_qonoff_sem);
 648         RETURN(0);
 649 }
 650
 651 static int mds_quota_check(struct obd_export *exp, unsigned int uid,
 652                            unsigned int gid, int inodes, int pending[2],
 653                            quota_acquire acquire, struct obd_trans_info *oti,
 654                            struct inode *inode, int frags)
 655 {
 656         return quota_is_set(exp->exp_obd, uid, gid, QI_SET) ?
 657                 quota_chk_acq_common(exp, uid, gid, inodes, pending, 0,
 658                                      acquire, oti, inode, frags) : 0;
 659 }
 660
 661 static int mds_quota_acquire(struct obd_device *obd, unsigned int uid,
 662                              unsigned int gid, struct obd_trans_info *oti)
 663 {
 664         struct lustre_quota_ctxt *qctxt = &obd->u.obt.obt_qctxt;
 665         int rc;
 666         ENTRY;
 667
 668         rc = qctxt_adjust_qunit(obd, qctxt, uid, gid, 0, 1, oti);
 669         RETURN(rc);
 670 }
 671
 672 static int mds_quota_pending_commit(struct obd_device *obd, unsigned int uid,
 673                                     unsigned int gid, int pending[2])
 674 {
 675         return quota_pending_commit(obd, uid, gid, pending, 0);
 676 }
 677 #endif /* HAVE_QUOTA_SUPPORT */
 678 #endif /* __KERNEL__ */
 679
 680 struct osc_quota_info {
 681         struct list_head        oqi_hash;       /* hash list */
 682         struct client_obd      *oqi_cli;        /* osc obd */
 683         unsigned int            oqi_id;         /* uid/gid of a file */
 684         short                   oqi_type;       /* quota type */
 685 };
 686
 687 spinlock_t qinfo_list_lock = SPIN_LOCK_UNLOCKED;
 688
 689 static struct list_head qinfo_hash[NR_DQHASH];
 690 /* SLAB cache for client quota context */
 691 cfs_mem_cache_t *qinfo_cachep = NULL;
 692
 693 static inline int hashfn(struct client_obd *cli, unsigned long id, int type)
 694                          __attribute__((__const__));
 695
 696 static inline int hashfn(struct client_obd *cli, unsigned long id, int type)
 697 {
 698         unsigned long tmp = ((unsigned long)cli>>6) ^ id;
 699         tmp = (tmp * (MAXQUOTAS - type)) % NR_DQHASH;
 700         return tmp;
 701 }
 702
 703 /* caller must hold qinfo_list_lock */
 704 static inline void insert_qinfo_hash(struct osc_quota_info *oqi)
 705 {
 706         struct list_head *head = qinfo_hash +
 707                 hashfn(oqi->oqi_cli, oqi->oqi_id, oqi->oqi_type);
 708
 709         LASSERT_SPIN_LOCKED(&qinfo_list_lock);
 710         list_add(&oqi->oqi_hash, head);
 711 }
 712
 713 /* caller must hold qinfo_list_lock */
 714 static inline void remove_qinfo_hash(struct osc_quota_info *oqi)
 715 {
 716         LASSERT_SPIN_LOCKED(&qinfo_list_lock);
 717         list_del_init(&oqi->oqi_hash);
 718 }
 719
 720 /* caller must hold qinfo_list_lock */
 721 static inline struct osc_quota_info *find_qinfo(struct client_obd *cli,
 722                                                 unsigned int id, int type)
 723 {
 724         unsigned int hashent = hashfn(cli, id, type);
 725         struct osc_quota_info *oqi;
 726
 727         LASSERT_SPIN_LOCKED(&qinfo_list_lock);
 728         list_for_each_entry(oqi, &qinfo_hash[hashent], oqi_hash) {
 729                 if (oqi->oqi_cli == cli &&
 730                     oqi->oqi_id == id && oqi->oqi_type == type)
 731                         return oqi;
 732         }
 733         return NULL;
 734 }
 735
 736 static struct osc_quota_info *alloc_qinfo(struct client_obd *cli,
 737                                           unsigned int id, int type)
 738 {
 739         struct osc_quota_info *oqi;
 740         ENTRY;
 741
 742         OBD_SLAB_ALLOC(oqi, qinfo_cachep, CFS_ALLOC_STD, sizeof(*oqi));
 743         if(!oqi)
 744                 RETURN(NULL);
 745
 746         CFS_INIT_LIST_HEAD(&oqi->oqi_hash);
 747         oqi->oqi_cli = cli;
 748         oqi->oqi_id = id;
 749         oqi->oqi_type = type;
 750
 751         RETURN(oqi);
 752 }
 753
 754 static void free_qinfo(struct osc_quota_info *oqi)
 755 {
 756         OBD_SLAB_FREE(oqi, qinfo_cachep, sizeof(*oqi));
 757 }
 758
 759 int osc_quota_chkdq(struct client_obd *cli, unsigned int uid, unsigned int gid)
 760 {
 761         unsigned int id;
 762         int cnt, rc = QUOTA_OK;
 763         ENTRY;
 764
 765         spin_lock(&qinfo_list_lock);
 766         for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
 767                 struct osc_quota_info *oqi = NULL;
 768
 769                 id = (cnt == USRQUOTA) ? uid : gid;
 770                 oqi = find_qinfo(cli, id, cnt);
 771                 if (oqi) {
 772                         rc = NO_QUOTA;
 773                         break;
 774                 }
 775         }
 776         spin_unlock(&qinfo_list_lock);
 777
 778         RETURN(rc);
 779 }
 780
 781 int osc_quota_setdq(struct client_obd *cli, unsigned int uid, unsigned int gid,
 782                     obd_flag valid, obd_flag flags)
 783 {
 784         unsigned int id;
 785         obd_flag noquota;
 786         int cnt, rc = 0;
 787         ENTRY;
 788
 789
 790         for (cnt = 0; cnt < MAXQUOTAS; cnt++) {
 791                 struct osc_quota_info *oqi, *old;
 792
 793                 if (!(valid & ((cnt == USRQUOTA) ?
 794                     OBD_MD_FLUSRQUOTA : OBD_MD_FLGRPQUOTA)))
 795                         continue;
 796
 797                 id = (cnt == USRQUOTA) ? uid : gid;
 798                 noquota = (cnt == USRQUOTA) ?
 799                     (flags & OBD_FL_NO_USRQUOTA) : (flags & OBD_FL_NO_GRPQUOTA);
 800
 801                 oqi = alloc_qinfo(cli, id, cnt);
 802                 if (!oqi) {
 803                         rc = -ENOMEM;
 804                         break;
 805                 }
 806
 807                 spin_lock(&qinfo_list_lock);
 808                 old = find_qinfo(cli, id, cnt);
 809                 if (old && !noquota)
 810                         remove_qinfo_hash(old);
 811                 else if (!old && noquota)
 812                         insert_qinfo_hash(oqi);
 813                 spin_unlock(&qinfo_list_lock);
 814
 815                 if (old || !noquota)
 816                         free_qinfo(oqi);
 817                 if (old && !noquota)
 818                         free_qinfo(old);
 819         }
 820
 821         RETURN(rc);
 822 }
 823
 824 int osc_quota_cleanup(struct obd_device *obd)
 825 {
 826         struct client_obd *cli = &obd->u.cli;
 827         struct osc_quota_info *oqi, *n;
 828         int i;
 829         ENTRY;
 830
 831         spin_lock(&qinfo_list_lock);
 832         for (i = 0; i < NR_DQHASH; i++) {
 833                 list_for_each_entry_safe(oqi, n, &qinfo_hash[i], oqi_hash) {
 834                         if (oqi->oqi_cli != cli)
 835                                 continue;
 836                         remove_qinfo_hash(oqi);
 837                         free_qinfo(oqi);
 838                 }
 839         }
 840         spin_unlock(&qinfo_list_lock);
 841
 842         RETURN(0);
 843 }
 844
 845 int osc_quota_init(void)
 846 {
 847         int i;
 848         ENTRY;
 849
 850         LASSERT(qinfo_cachep == NULL);
 851         qinfo_cachep = cfs_mem_cache_create("osc_quota_info",
 852                                             sizeof(struct osc_quota_info),
 853                                             0, 0);
 854         if (!qinfo_cachep)
 855                 RETURN(-ENOMEM);
 856
 857         for (i = 0; i < NR_DQHASH; i++)
 858                 CFS_INIT_LIST_HEAD(qinfo_hash + i);
 859
 860         RETURN(0);
 861 }
 862
 863 int osc_quota_exit(void)
 864 {
 865         struct osc_quota_info *oqi, *n;
 866         int i, rc;
 867         ENTRY;
 868
 869         spin_lock(&qinfo_list_lock);
 870         for (i = 0; i < NR_DQHASH; i++) {
 871                 list_for_each_entry_safe(oqi, n, &qinfo_hash[i], oqi_hash) {
 872                         remove_qinfo_hash(oqi);
 873                         free_qinfo(oqi);
 874                 }
 875         }
 876         spin_unlock(&qinfo_list_lock);
 877
 878         rc = cfs_mem_cache_destroy(qinfo_cachep);
 879         LASSERTF(rc == 0, "couldn't destory qinfo_cachep slab\n");
 880         qinfo_cachep = NULL;
 881
 882         RETURN(0);
 883 }
 884
 885 #ifdef __KERNEL__
 886 #ifdef HAVE_QUOTA_SUPPORT
 887 quota_interface_t mds_quota_interface = {
 888         .quota_init     = mds_quota_init,
 889         .quota_exit     = mds_quota_exit,
 890         .quota_setup    = mds_quota_setup,
 891         .quota_cleanup  = mds_quota_cleanup,
 892         .quota_check    = target_quota_check,
 893         .quota_ctl      = mds_quota_ctl,
 894         .quota_fs_cleanup       =mds_quota_fs_cleanup,
 895         .quota_recovery = mds_quota_recovery,
 896         .quota_adjust   = mds_quota_adjust,
 897         .quota_chkquota = mds_quota_check,
 898         .quota_acquire  = mds_quota_acquire,
 899         .quota_pending_commit = mds_quota_pending_commit,
 900 };
 901
 902 quota_interface_t filter_quota_interface = {
 903         .quota_setup    = filter_quota_setup,
 904         .quota_cleanup  = filter_quota_cleanup,
 905         .quota_check    = target_quota_check,
 906         .quota_ctl      = filter_quota_ctl,
 907         .quota_setinfo  = filter_quota_setinfo,
 908         .quota_clearinfo = filter_quota_clearinfo,
 909         .quota_enforce  = filter_quota_enforce,
 910         .quota_getflag  = filter_quota_getflag,
 911         .quota_acquire  = filter_quota_acquire,
 912         .quota_adjust   = filter_quota_adjust,
 913         .quota_chkquota = filter_quota_check,
 914         .quota_adjust_qunit   = filter_quota_adjust_qunit,
 915         .quota_pending_commit = filter_quota_pending_commit,
 916 };
 917 #endif
 918 #endif /* __KERNEL__ */
 919
 920 quota_interface_t mdc_quota_interface = {
 921         .quota_ctl      = client_quota_ctl,
 922         .quota_check    = client_quota_check,
 923         .quota_poll_check = client_quota_poll_check,
 924 };
 925
 926 quota_interface_t osc_quota_interface = {
 927         .quota_ctl      = client_quota_ctl,
 928         .quota_check    = client_quota_check,
 929         .quota_poll_check = client_quota_poll_check,
 930         .quota_init     = osc_quota_init,
 931         .quota_exit     = osc_quota_exit,
 932         .quota_chkdq    = osc_quota_chkdq,
 933         .quota_setdq    = osc_quota_setdq,
 934         .quota_cleanup  = osc_quota_cleanup,
 935         .quota_adjust_qunit = client_quota_adjust_qunit,
 936 };
 937
 938 quota_interface_t lov_quota_interface = {
 939         .quota_check    = lov_quota_check,
 940         .quota_ctl      = lov_quota_ctl,
 941         .quota_adjust_qunit = lov_quota_adjust_qunit,
 942 };
 943
 944 #ifdef __KERNEL__
 945
 946 cfs_proc_dir_entry_t *lquota_type_proc_dir = NULL;
 947
 948 static int __init init_lustre_quota(void)
 949 {
 950 #ifdef HAVE_QUOTA_SUPPORT
 951         int rc = 0;
 952
 953         lquota_type_proc_dir = lprocfs_register(OBD_LQUOTA_DEVICENAME,
 954                                                 proc_lustre_root,
 955                                                 NULL, NULL);
 956         if (IS_ERR(lquota_type_proc_dir)) {
 957                 CERROR("LProcFS failed in lquota-init\n");
 958                 rc = PTR_ERR(lquota_type_proc_dir);
 959                 return rc;
 960         }
 961
 962         rc = qunit_cache_init();
 963         if (rc)
 964                 return rc;
 965
 966         PORTAL_SYMBOL_REGISTER(filter_quota_interface);
 967         PORTAL_SYMBOL_REGISTER(mds_quota_interface);
 968 #endif
 969         PORTAL_SYMBOL_REGISTER(mdc_quota_interface);
 970         PORTAL_SYMBOL_REGISTER(osc_quota_interface);
 971         PORTAL_SYMBOL_REGISTER(lov_quota_interface);
 972         return 0;
 973 }
 974
 975 static void /*__exit*/ exit_lustre_quota(void)
 976 {
 977         PORTAL_SYMBOL_UNREGISTER(mdc_quota_interface);
 978         PORTAL_SYMBOL_UNREGISTER(osc_quota_interface);
 979         PORTAL_SYMBOL_UNREGISTER(lov_quota_interface);
 980 #ifdef HAVE_QUOTA_SUPPORT
 981         PORTAL_SYMBOL_UNREGISTER(filter_quota_interface);
 982         PORTAL_SYMBOL_UNREGISTER(mds_quota_interface);
 983
 984         qunit_cache_cleanup();
 985
 986         if (lquota_type_proc_dir)
 987                 lprocfs_remove(&lquota_type_proc_dir);
 988 #endif
 989 }
 990
 991 MODULE_AUTHOR("Sun Microsystems, Inc. <http://www.lustre.org/>");
 992 MODULE_DESCRIPTION("Lustre Quota");
 993 MODULE_LICENSE("GPL");
 994
 995 cfs_module(lquota, "1.0.0", init_lustre_quota, exit_lustre_quota);
 996
 997 #ifdef HAVE_QUOTA_SUPPORT
 998 EXPORT_SYMBOL(mds_quota_interface);
 999 EXPORT_SYMBOL(filter_quota_interface);
1000 #endif
1001 EXPORT_SYMBOL(mdc_quota_interface);
1002 EXPORT_SYMBOL(osc_quota_interface);
1003 EXPORT_SYMBOL(lov_quota_interface);
1004 #endif /* __KERNEL */