lustre/quota/qsd_reint.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; if not, write to the
  18  * Free Software Foundation, Inc., 59 Temple Place - Suite 330,
  19  * Boston, MA 021110-1307, USA
  20  *
  21  * GPL HEADER END
  22  */
  23 /*
  24  * Copyright (c) 2012, 2014, Intel Corporation.
  25  * Use is subject to license terms.
  26  *
  27  * Author: Johann Lombardi <johann.lombardi@intel.com>
  28  * Author: Niu    Yawei    <yawei.niu@intel.com>
  29  */
  30
  31 #define DEBUG_SUBSYSTEM S_LQUOTA
  32
  33 #include <linux/kthread.h>
  34 #include "qsd_internal.h"
  35
  36 /*
  37  * Completion function invoked when the global quota lock enqueue has completed
  38  */
  39 static void qsd_reint_completion(const struct lu_env *env,
  40                                  struct qsd_qtype_info *qqi,
  41                                  struct quota_body *req_qbody,
  42                                  struct quota_body *rep_qbody,
  43                                  struct lustre_handle *lockh,
  44                                  struct lquota_lvb *lvb,
  45                                  void *arg, int rc)
  46 {
  47         struct qsd_instance     *qsd = qqi->qqi_qsd;
  48         __u64                   *slv_ver = (__u64 *)arg;
  49         ENTRY;
  50
  51         if (rc) {
  52                 CDEBUG_LIMIT(rc != -EAGAIN ? D_ERROR : D_QUOTA,
  53                              "%s: failed to enqueue global quota lock, glb fid:"
  54                              DFID", rc:%d\n", qsd->qsd_svname,
  55                              PFID(&req_qbody->qb_fid), rc);
  56                 RETURN_EXIT;
  57         }
  58
  59         CDEBUG(D_QUOTA, "%s: global quota lock successfully acquired, glb "
  60                "fid:"DFID", glb ver:"LPU64", slv fid:"DFID", slv ver:"LPU64"\n",
  61                qsd->qsd_svname, PFID(&req_qbody->qb_fid),
  62                lvb->lvb_glb_ver, PFID(&rep_qbody->qb_slv_fid),
  63                rep_qbody->qb_slv_ver);
  64
  65         *slv_ver = rep_qbody->qb_slv_ver;
  66         memcpy(&qqi->qqi_slv_fid, &rep_qbody->qb_slv_fid,
  67                sizeof(struct lu_fid));
  68         lustre_handle_copy(&qqi->qqi_lockh, lockh);
  69         EXIT;
  70 }
  71
  72 static int qsd_reint_qid(const struct lu_env *env, struct qsd_qtype_info *qqi,
  73                          bool global, union lquota_id *qid, void *rec)
  74 {
  75         struct lquota_entry     *lqe;
  76         int                      rc;
  77         ENTRY;
  78
  79         lqe = lqe_locate(env, qqi->qqi_site, qid);
  80         if (IS_ERR(lqe))
  81                 RETURN(PTR_ERR(lqe));
  82
  83         LQUOTA_DEBUG(lqe, "reintegrating entry");
  84
  85         rc = qsd_update_lqe(env, lqe, global, rec);
  86         if (rc)
  87                 GOTO(out, rc);
  88
  89         rc = qsd_update_index(env, qqi, qid, global, 0, rec);
  90 out:
  91         lqe_putref(lqe);
  92         RETURN(rc);
  93 }
  94
  95 static int qsd_reint_entries(const struct lu_env *env,
  96                              struct qsd_qtype_info *qqi,
  97                              struct idx_info *ii, bool global,
  98                              struct page **pages,
  99                              unsigned int npages, bool need_swab)
 100 {
 101         struct qsd_thread_info  *qti = qsd_info(env);
 102         struct qsd_instance     *qsd = qqi->qqi_qsd;
 103         union lquota_id         *qid = &qti->qti_id;
 104         int                      i, j, k, size;
 105         int                      rc = 0;
 106         ENTRY;
 107
 108         CDEBUG(D_QUOTA, "%s: processing %d pages for %s index\n",
 109                qsd->qsd_svname, npages, global ? "global" : "slave");
 110
 111         /* sanity check on the record size */
 112         if ((global && ii->ii_recsize != sizeof(struct lquota_glb_rec)) ||
 113             (!global && ii->ii_recsize != sizeof(struct lquota_slv_rec))) {
 114                 CERROR("%s: invalid record size (%d) for %s index\n",
 115                        qsd->qsd_svname, ii->ii_recsize,
 116                        global ? "global" : "slave");
 117                 RETURN(-EINVAL);
 118         }
 119
 120         size = ii->ii_recsize + ii->ii_keysize;
 121
 122         for (i = 0; i < npages; i++) {
 123                 union lu_page   *lip = kmap(pages[i]);
 124
 125                 for (j = 0; j < LU_PAGE_COUNT; j++) {
 126                         if (need_swab)
 127                                 /* swab header */
 128                                 lustre_swab_lip_header(&lip->lp_idx);
 129
 130                         if (lip->lp_idx.lip_magic != LIP_MAGIC) {
 131                                 CERROR("%s: invalid magic (%x != %x) for page "
 132                                        "%d/%d while transferring %s index\n",
 133                                        qsd->qsd_svname, lip->lp_idx.lip_magic,
 134                                        LIP_MAGIC, i + 1, npages,
 135                                        global ? "global" : "slave");
 136                                 GOTO(out, rc = -EINVAL);
 137                         }
 138
 139                         CDEBUG(D_QUOTA, "%s: processing page %d/%d with %d "
 140                                "entries for %s index\n", qsd->qsd_svname, i + 1,
 141                                npages, lip->lp_idx.lip_nr,
 142                                global ? "global" : "slave");
 143
 144                         for (k = 0; k < lip->lp_idx.lip_nr; k++) {
 145                                 char *entry;
 146
 147                                 entry = lip->lp_idx.lip_entries + k * size;
 148                                 memcpy(qid, entry, ii->ii_keysize); /* key */
 149                                 entry += ii->ii_keysize;            /* value */
 150
 151                                 if (need_swab) {
 152                                         int offset = 0;
 153
 154                                         /* swab key */
 155                                         __swab64s(&qid->qid_uid);
 156                                         /* quota records only include 64-bit
 157                                          * fields */
 158                                         while (offset < ii->ii_recsize) {
 159                                                 __swab64s((__u64 *)
 160                                                               (entry + offset));
 161                                                 offset += sizeof(__u64);
 162                                         }
 163                                 }
 164
 165                                 rc = qsd_reint_qid(env, qqi, global, qid,
 166                                                    (void *)entry);
 167                                 if (rc)
 168                                         GOTO(out, rc);
 169                         }
 170                         lip++;
 171                 }
 172 out:
 173                 kunmap(pages[i]);
 174                 if (rc)
 175                         break;
 176         }
 177         RETURN(rc);
 178 }
 179
 180 static int qsd_reint_index(const struct lu_env *env, struct qsd_qtype_info *qqi,
 181                            bool global)
 182 {
 183         struct qsd_thread_info  *qti = qsd_info(env);
 184         struct qsd_instance     *qsd = qqi->qqi_qsd;
 185         struct idx_info         *ii = &qti->qti_ii;
 186         struct lu_fid           *fid;
 187         struct page             **pages = NULL;
 188         unsigned int             npages, pg_cnt;
 189         __u64                    start_hash = 0, ver = 0;
 190         bool                     need_swab = false;
 191         int                      i, rc;
 192         ENTRY;
 193
 194         fid = global ? &qqi->qqi_fid : &qqi->qqi_slv_fid;
 195
 196         /* let's do a 1MB bulk */
 197         npages = min_t(unsigned int, OFD_MAX_BRW_SIZE, 1 << 20);
 198         npages /= PAGE_CACHE_SIZE;
 199
 200         /* allocate pages for bulk index read */
 201         OBD_ALLOC(pages, npages * sizeof(*pages));
 202         if (pages == NULL)
 203                 GOTO(out, rc = -ENOMEM);
 204         for (i = 0; i < npages; i++) {
 205                 pages[i] = alloc_page(GFP_IOFS);
 206                 if (pages[i] == NULL)
 207                         GOTO(out, rc = -ENOMEM);
 208         }
 209
 210 repeat:
 211         /* initialize index_info request with FID of global index */
 212         memset(ii, 0, sizeof(*ii));
 213         memcpy(&ii->ii_fid, fid, sizeof(*fid));
 214         ii->ii_magic = IDX_INFO_MAGIC;
 215         ii->ii_flags = II_FL_NOHASH;
 216         ii->ii_count = npages * LU_PAGE_COUNT;
 217         ii->ii_hash_start = start_hash;
 218
 219         /* send bulk request to quota master to read global index */
 220         rc = qsd_fetch_index(env, qsd->qsd_exp, ii, npages, pages, &need_swab);
 221         if (rc) {
 222                 CWARN("%s: failed to fetch index for "DFID". %d\n",
 223                       qsd->qsd_svname, PFID(fid), rc);
 224                 GOTO(out, rc);
 225         }
 226
 227         /* various sanity checks */
 228         if (ii->ii_magic != IDX_INFO_MAGIC) {
 229                 CERROR("%s: invalid magic in index transfer %x != %x\n",
 230                        qsd->qsd_svname, ii->ii_magic, IDX_INFO_MAGIC);
 231                 GOTO(out, rc = -EPROTO);
 232         }
 233         if ((ii->ii_flags & II_FL_VARKEY) != 0)
 234                 CWARN("%s: II_FL_VARKEY is set on index transfer for fid "DFID
 235                       ", it shouldn't be\n", qsd->qsd_svname, PFID(fid));
 236         if ((ii->ii_flags & II_FL_NONUNQ) != 0)
 237                 CWARN("%s: II_FL_NONUNQ is set on index transfer for fid "DFID
 238                       ", it shouldn't be\n", qsd->qsd_svname, PFID(fid));
 239         if (ii->ii_keysize != sizeof(__u64)) {
 240                 CERROR("%s: invalid key size reported on index transfer for "
 241                        "fid "DFID", %u != %u\n", qsd->qsd_svname, PFID(fid),
 242                        ii->ii_keysize, (int)sizeof(__u64));
 243                 GOTO(out, rc = -EPROTO);
 244         }
 245         if (ii->ii_version == 0 && ii->ii_count != 0)
 246                 CWARN("%s: index version for fid "DFID" is 0, but index isn't "
 247                       "empty (%d)\n", qsd->qsd_svname, PFID(fid), ii->ii_count);
 248
 249         CDEBUG(D_QUOTA, "%s: reintegration process for fid "DFID" successfully "
 250                "fetched %s index, count = %d\n", qsd->qsd_svname,
 251                PFID(fid), global ? "global" : "slave", ii->ii_count);
 252
 253         if (start_hash == 0)
 254                 /* record version associated with the first bulk transfer */
 255                 ver = ii->ii_version;
 256
 257         pg_cnt = (ii->ii_count + (LU_PAGE_COUNT) - 1);
 258         pg_cnt >>= PAGE_CACHE_SHIFT - LU_PAGE_SHIFT;
 259
 260         if (pg_cnt > npages) {
 261                 CERROR("%s: master returned more pages than expected, %u > %u"
 262                        "\n", qsd->qsd_svname, pg_cnt, npages);
 263                 pg_cnt = npages;
 264         }
 265
 266         rc = qsd_reint_entries(env, qqi, ii, global, pages, pg_cnt, need_swab);
 267         if (rc)
 268                 GOTO(out, rc);
 269
 270         if (ii->ii_hash_end != II_END_OFF) {
 271                 start_hash = ii->ii_hash_end;
 272                 goto repeat;
 273         }
 274 out:
 275         if (pages != NULL) {
 276                 for (i = 0; i < npages; i++)
 277                         if (pages[i] != NULL)
 278                                 __free_page(pages[i]);
 279                 OBD_FREE(pages, npages * sizeof(*pages));
 280         }
 281
 282         /* Update index version */
 283         if (rc == 0) {
 284                 rc = qsd_write_version(env, qqi, ver, global);
 285                 if (rc)
 286                         CERROR("%s: write version "LPU64" to "DFID" failed. "
 287                                "%d\n", qsd->qsd_svname, ver, PFID(fid), rc);
 288         }
 289
 290         RETURN(rc);
 291 }
 292
 293 static int qsd_reconciliation(const struct lu_env *env,
 294                               struct qsd_qtype_info *qqi)
 295 {
 296         struct qsd_thread_info  *qti = qsd_info(env);
 297         struct qsd_instance     *qsd = qqi->qqi_qsd;
 298         const struct dt_it_ops  *iops;
 299         struct dt_it            *it;
 300         struct dt_key           *key;
 301         struct lquota_entry     *lqe;
 302         union lquota_id         *qid = &qti->qti_id;
 303         int                      rc;
 304         ENTRY;
 305
 306         LASSERT(qqi->qqi_glb_obj != NULL);
 307         iops = &qqi->qqi_glb_obj->do_index_ops->dio_it;
 308
 309         it = iops->init(env, qqi->qqi_glb_obj, 0);
 310         if (IS_ERR(it)) {
 311                 CWARN("%s: Initialize it for "DFID" failed. %ld\n",
 312                       qsd->qsd_svname, PFID(&qqi->qqi_fid), PTR_ERR(it));
 313                 RETURN(PTR_ERR(it));
 314         }
 315
 316         rc = iops->load(env, it, 0);
 317         if (rc < 0) {
 318                 CWARN("%s: Load first entry for "DFID" failed. %d\n",
 319                       qsd->qsd_svname, PFID(&qqi->qqi_fid), rc);
 320                 GOTO(out, rc);
 321         } else if (rc == 0) {
 322                 rc = iops->next(env, it);
 323                 if (rc != 0)
 324                         GOTO(out, rc = (rc < 0) ? rc : 0);
 325         }
 326
 327         do {
 328                 key = iops->key(env, it);
 329                 if (IS_ERR(key)) {
 330                         CWARN("%s: Error key for "DFID". %ld\n",
 331                               qsd->qsd_svname, PFID(&qqi->qqi_fid),
 332                               PTR_ERR(key));
 333                         GOTO(out, rc = PTR_ERR(key));
 334                 }
 335
 336                 /* skip the root user/group */
 337                 if (*((__u64 *)key) == 0)
 338                         goto next;
 339
 340                 qid->qid_uid = *((__u64 *)key);
 341
 342                 lqe = lqe_locate(env, qqi->qqi_site, qid);
 343                 if (IS_ERR(lqe)) {
 344                         CWARN("%s: failed to locate lqe. "DFID", %ld\n",
 345                               qsd->qsd_svname, PFID(&qqi->qqi_fid),
 346                               PTR_ERR(lqe));
 347                         GOTO(out, rc = PTR_ERR(lqe));
 348                 }
 349
 350                 rc = qsd_refresh_usage(env, lqe);
 351                 if (rc) {
 352                         CWARN("%s: failed to get usage. "DFID", %d\n",
 353                               qsd->qsd_svname, PFID(&qqi->qqi_fid), rc);
 354                         lqe_putref(lqe);
 355                         GOTO(out, rc);
 356                 }
 357
 358                 rc = qsd_adjust(env, lqe);
 359                 lqe_putref(lqe);
 360                 if (rc) {
 361                         CWARN("%s: failed to report quota. "DFID", %d\n",
 362                               qsd->qsd_svname, PFID(&qqi->qqi_fid), rc);
 363                         GOTO(out, rc);
 364                 }
 365 next:
 366                 rc = iops->next(env, it);
 367                 if (rc < 0)
 368                         CWARN("%s: failed to parse index, ->next error:%d "DFID
 369                               "\n", qsd->qsd_svname, rc, PFID(&qqi->qqi_fid));
 370         } while (rc == 0);
 371
 372         /* reach the end */
 373         if (rc > 0)
 374                 rc = 0;
 375 out:
 376         iops->put(env, it);
 377         iops->fini(env, it);
 378         RETURN(rc);
 379 }
 380
 381 static int qsd_connected(struct qsd_instance *qsd)
 382 {
 383         int     connected;
 384
 385         read_lock(&qsd->qsd_lock);
 386         connected = qsd->qsd_exp_valid ? 1 : 0;
 387         read_unlock(&qsd->qsd_lock);
 388
 389         return connected;
 390 }
 391
 392 static int qsd_started(struct qsd_instance *qsd)
 393 {
 394         int     started;
 395
 396         read_lock(&qsd->qsd_lock);
 397         started = qsd->qsd_started ? 1 : 0;
 398         read_unlock(&qsd->qsd_lock);
 399
 400         return started;
 401 }
 402
 403 /*
 404  * Routine executed by the reintegration thread.
 405  */
 406 static int qsd_reint_main(void *args)
 407 {
 408         struct lu_env           *env;
 409         struct qsd_thread_info  *qti;
 410         struct qsd_qtype_info   *qqi = (struct qsd_qtype_info *)args;
 411         struct qsd_instance     *qsd = qqi->qqi_qsd;
 412         struct ptlrpc_thread    *thread = &qqi->qqi_reint_thread;
 413         struct l_wait_info       lwi = { 0 };
 414         int                      rc;
 415         ENTRY;
 416
 417         CDEBUG(D_QUOTA, "%s: Starting reintegration thread for "DFID"\n",
 418                qsd->qsd_svname, PFID(&qqi->qqi_fid));
 419
 420         qqi_getref(qqi);
 421         lu_ref_add(&qqi->qqi_reference, "reint_thread", thread);
 422
 423         thread_set_flags(thread, SVC_RUNNING);
 424         wake_up(&thread->t_ctl_waitq);
 425
 426         OBD_ALLOC_PTR(env);
 427         if (env == NULL)
 428                 GOTO(out, rc = -ENOMEM);
 429
 430         /* initialize environment */
 431         rc = lu_env_init(env, LCT_DT_THREAD);
 432         if (rc)
 433                 GOTO(out_env, rc);
 434         qti = qsd_info(env);
 435
 436         /* wait for the connection to master established */
 437         l_wait_event(thread->t_ctl_waitq,
 438                      qsd_connected(qsd) || !thread_is_running(thread), &lwi);
 439
 440         /* Step 1: enqueue global index lock */
 441         if (!thread_is_running(thread))
 442                 GOTO(out_env_init, rc = 0);
 443
 444         LASSERT(qsd->qsd_exp != NULL);
 445         LASSERT(qqi->qqi_glb_uptodate == 0 || qqi->qqi_slv_uptodate == 0);
 446
 447         memset(&qti->qti_lvb, 0, sizeof(qti->qti_lvb));
 448
 449         read_lock(&qsd->qsd_lock);
 450         /* check whether we already own a global quota lock for this type */
 451         if (lustre_handle_is_used(&qqi->qqi_lockh) &&
 452             ldlm_lock_addref_try(&qqi->qqi_lockh, qsd_glb_einfo.ei_mode) == 0) {
 453                 read_unlock(&qsd->qsd_lock);
 454                 /* force refresh of global & slave index copy */
 455                 qti->qti_lvb.lvb_glb_ver = ~0ULL;
 456                 qti->qti_slv_ver = ~0ULL;
 457         } else {
 458                 /* no valid lock found, let's enqueue a new one */
 459                 read_unlock(&qsd->qsd_lock);
 460
 461                 memset(&qti->qti_body, 0, sizeof(qti->qti_body));
 462                 memcpy(&qti->qti_body.qb_fid, &qqi->qqi_fid,
 463                        sizeof(qqi->qqi_fid));
 464
 465                 rc = qsd_intent_lock(env, qsd->qsd_exp, &qti->qti_body, true,
 466                                      IT_QUOTA_CONN, qsd_reint_completion, qqi,
 467                                      &qti->qti_lvb, (void *)&qti->qti_slv_ver);
 468                 if (rc)
 469                         GOTO(out_env_init, rc);
 470
 471                 CDEBUG(D_QUOTA, "%s: glb_ver:"LPU64"/"LPU64",slv_ver:"LPU64"/"
 472                        LPU64"\n", qsd->qsd_svname,
 473                        qti->qti_lvb.lvb_glb_ver, qqi->qqi_glb_ver,
 474                        qti->qti_slv_ver, qqi->qqi_slv_ver);
 475         }
 476
 477         /* Step 2: reintegrate global index */
 478         if (!thread_is_running(thread))
 479                 GOTO(out_lock, rc = 0);
 480
 481         OBD_FAIL_TIMEOUT(OBD_FAIL_QUOTA_DELAY_REINT, 10);
 482
 483         if (qqi->qqi_glb_ver != qti->qti_lvb.lvb_glb_ver) {
 484                 rc = qsd_reint_index(env, qqi, true);
 485                 if (rc) {
 486                         CWARN("%s: reint global for "DFID" failed. %d\n",
 487                               qsd->qsd_svname, PFID(&qqi->qqi_fid), rc);
 488                         GOTO(out_lock, rc);
 489                 }
 490         } else {
 491                 qsd_bump_version(qqi, qqi->qqi_glb_ver, true);
 492         }
 493
 494         /* Step 3: reintegrate slave index */
 495         if (!thread_is_running(thread))
 496                 GOTO(out_lock, rc = 0);
 497
 498         if (qqi->qqi_slv_ver != qti->qti_slv_ver) {
 499                 rc = qsd_reint_index(env, qqi, false);
 500                 if (rc) {
 501                         CWARN("%s: reintegration for "DFID" failed with %d\n",
 502                               qsd->qsd_svname, PFID(&qqi->qqi_slv_fid), rc);
 503                         GOTO(out_lock, rc);
 504                 }
 505         } else {
 506                 qsd_bump_version(qqi, qqi->qqi_slv_ver, false);
 507         }
 508
 509         /* wait for the qsd instance started (target recovery done) */
 510         l_wait_event(thread->t_ctl_waitq,
 511                      qsd_started(qsd) || !thread_is_running(thread), &lwi);
 512
 513         if (!thread_is_running(thread))
 514                 GOTO(out_lock, rc = 0);
 515
 516         /* Step 4: start reconciliation for each enforced ID */
 517         rc = qsd_reconciliation(env, qqi);
 518         if (rc)
 519                 CWARN("%s: reconciliation for "DFID" failed with %d\n",
 520                       qsd->qsd_svname, PFID(&qqi->qqi_slv_fid), rc);
 521
 522         EXIT;
 523 out_lock:
 524         ldlm_lock_decref(&qqi->qqi_lockh, qsd_glb_einfo.ei_mode);
 525 out_env_init:
 526         lu_env_fini(env);
 527 out_env:
 528         OBD_FREE_PTR(env);
 529 out:
 530         write_lock(&qsd->qsd_lock);
 531         qqi->qqi_reint = 0;
 532         write_unlock(&qsd->qsd_lock);
 533
 534         qqi_putref(qqi);
 535         lu_ref_del(&qqi->qqi_reference, "reint_thread", thread);
 536
 537         thread_set_flags(thread, SVC_STOPPED);
 538         wake_up(&thread->t_ctl_waitq);
 539         return rc;
 540 }
 541
 542 void qsd_stop_reint_thread(struct qsd_qtype_info *qqi)
 543 {
 544         struct ptlrpc_thread    *thread = &qqi->qqi_reint_thread;
 545         struct l_wait_info       lwi = { 0 };
 546
 547         if (!thread_is_stopped(thread)) {
 548                 thread_set_flags(thread, SVC_STOPPING);
 549                 wake_up(&thread->t_ctl_waitq);
 550
 551                 l_wait_event(thread->t_ctl_waitq,
 552                              thread_is_stopped(thread), &lwi);
 553         }
 554 }
 555
 556 static int qsd_entry_iter_cb(cfs_hash_t *hs, cfs_hash_bd_t *bd,
 557                              struct hlist_node *hnode, void *data)
 558 {
 559         struct lquota_entry     *lqe;
 560         int                     *pending = (int *)data;
 561
 562         lqe = hlist_entry(hnode, struct lquota_entry, lqe_hash);
 563         LASSERT(atomic_read(&lqe->lqe_ref) > 0);
 564
 565         lqe_read_lock(lqe);
 566         *pending += lqe->lqe_pending_req;
 567         lqe_read_unlock(lqe);
 568
 569         return 0;
 570 }
 571
 572 static bool qsd_pending_updates(struct qsd_qtype_info *qqi)
 573 {
 574         struct qsd_instance     *qsd = qqi->qqi_qsd;
 575         struct qsd_upd_rec      *upd;
 576         struct lquota_entry     *lqe, *n;
 577         int                      dqacq = 0;
 578         bool                     updates = false;
 579         ENTRY;
 580
 581         /* any pending quota adjust? */
 582         spin_lock(&qsd->qsd_adjust_lock);
 583         list_for_each_entry_safe(lqe, n, &qsd->qsd_adjust_list, lqe_link) {
 584                 if (lqe2qqi(lqe) == qqi) {
 585                         list_del_init(&lqe->lqe_link);
 586                         lqe_putref(lqe);
 587                 }
 588         }
 589         spin_unlock(&qsd->qsd_adjust_lock);
 590
 591         /* any pending updates? */
 592         read_lock(&qsd->qsd_lock);
 593         list_for_each_entry(upd, &qsd->qsd_upd_list, qur_link) {
 594                 if (upd->qur_qqi == qqi) {
 595                         read_unlock(&qsd->qsd_lock);
 596                         CDEBUG(D_QUOTA, "%s: pending %s updates for type:%d.\n",
 597                                qsd->qsd_svname,
 598                                upd->qur_global ? "global" : "slave",
 599                                qqi->qqi_qtype);
 600                         GOTO(out, updates = true);
 601                 }
 602         }
 603         read_unlock(&qsd->qsd_lock);
 604
 605         /* any pending quota request? */
 606         cfs_hash_for_each_safe(qqi->qqi_site->lqs_hash, qsd_entry_iter_cb,
 607                                &dqacq);
 608         if (dqacq) {
 609                 CDEBUG(D_QUOTA, "%s: pending dqacq for type:%d.\n",
 610                        qsd->qsd_svname, qqi->qqi_qtype);
 611                 updates = true;
 612         }
 613         EXIT;
 614 out:
 615         if (updates)
 616                 CERROR("%s: Delaying reintegration for qtype:%d until pending "
 617                        "updates are flushed.\n",
 618                        qsd->qsd_svname, qqi->qqi_qtype);
 619         return updates;
 620 }
 621
 622 int qsd_start_reint_thread(struct qsd_qtype_info *qqi)
 623 {
 624         struct ptlrpc_thread    *thread = &qqi->qqi_reint_thread;
 625         struct qsd_instance     *qsd = qqi->qqi_qsd;
 626         struct l_wait_info       lwi = { 0 };
 627         struct task_struct      *task;
 628         int                      rc;
 629         char                    *name;
 630         ENTRY;
 631
 632         /* don't bother to do reintegration when quota isn't enabled */
 633         if (!qsd_type_enabled(qsd, qqi->qqi_qtype))
 634                 RETURN(0);
 635
 636         if (qsd->qsd_acct_failed)
 637                 /* no space accounting support, can't enable enforcement */
 638                 RETURN(0);
 639
 640         /* check if the reintegration has already started or finished */
 641         write_lock(&qsd->qsd_lock);
 642
 643         if ((qqi->qqi_glb_uptodate && qqi->qqi_slv_uptodate) ||
 644              qqi->qqi_reint || qsd->qsd_stopping) {
 645                 write_unlock(&qsd->qsd_lock);
 646                 RETURN(0);
 647         }
 648         qqi->qqi_reint = 1;
 649
 650         write_unlock(&qsd->qsd_lock);
 651
 652         /* there could be some unfinished global or index entry updates
 653          * (very unlikely), to avoid them messing up with the reint
 654          * procedure, we just return and try to re-start reint later. */
 655         if (qsd_pending_updates(qqi)) {
 656                 write_lock(&qsd->qsd_lock);
 657                 qqi->qqi_reint = 0;
 658                 write_unlock(&qsd->qsd_lock);
 659                 RETURN(0);
 660         }
 661
 662         OBD_ALLOC(name, MTI_NAME_MAXLEN);
 663         if (name == NULL)
 664                 RETURN(-ENOMEM);
 665
 666         snprintf(name, MTI_NAME_MAXLEN, "qsd_reint_%d.%s",
 667                  qqi->qqi_qtype, qsd->qsd_svname);
 668
 669         task = kthread_run(qsd_reint_main, qqi, name);
 670         OBD_FREE(name, MTI_NAME_MAXLEN);
 671
 672         if (IS_ERR(task)) {
 673                 rc = PTR_ERR(task);
 674                 thread_set_flags(thread, SVC_STOPPED);
 675                 write_lock(&qsd->qsd_lock);
 676                 qqi->qqi_reint = 0;
 677                 write_unlock(&qsd->qsd_lock);
 678                 RETURN(rc);
 679         }
 680
 681         l_wait_event(thread->t_ctl_waitq,
 682                      thread_is_running(thread) || thread_is_stopped(thread),
 683                      &lwi);
 684         RETURN(0);
 685 }