From e97152e1766de834001f5c4cdb3a6e38b5ecb2f2 Mon Sep 17 00:00:00 2001
From: fanyong <fanyong>
Date: Thu, 30 Apr 2009 01:43:19 +0000
Subject: [PATCH] Branch HEAD b=19171 i=tianzy i=robert.read

Hold obd reference count when quota recovery.
---
 lustre/include/obd.h             |  2 ++
 lustre/include/obd_class.h       | 20 ++++++++++++++
 lustre/lov/lov_dev.c             |  8 +++---
 lustre/lov/lov_internal.h        |  3 +--
 lustre/lov/lov_log.c             | 12 ++++-----
 lustre/lov/lov_obd.c             | 57 ++++++++++++++++++++--------------------
 lustre/lov/lov_pool.c            |  8 +++---
 lustre/lov/lov_qos.c             |  4 +--
 lustre/mdt/mdt_handler.c         |  6 +++--
 lustre/obdclass/lprocfs_status.c |  2 ++
 lustre/quota/quota_ctl.c         |  2 ++
 lustre/quota/quota_master.c      |  6 +++--
 12 files changed, 80 insertions(+), 50 deletions(-)

diff --git a/lustre/include/obd.h b/lustre/include/obd.h
index 79c9fb2..f0e802c 100644
--- a/lustre/include/obd.h
+++ b/lustre/include/obd.h
@@ -1413,6 +1413,8 @@ struct obd_ops {
                           char *ostname);
         int (*o_pool_rem)(struct obd_device *obd, char *poolname,
                           char *ostname);
+        void (*o_getref)(struct obd_device *obd);
+        void (*o_putref)(struct obd_device *obd);
         /*
          * NOTE: If adding ops, add another LPROCFS_OBD_OP_INIT() line
          * to lprocfs_alloc_obd_stats() in obdclass/lprocfs_status.c.
diff --git a/lustre/include/obd_class.h b/lustre/include/obd_class.h
index b47d60b..dba5e2d 100644
--- a/lustre/include/obd_class.h
+++ b/lustre/include/obd_class.h
@@ -990,6 +990,26 @@ static inline int obd_pool_rem(struct obd_device *obd, char *poolname, char *ost
         RETURN(rc);
 }
 
+static inline void obd_getref(struct obd_device *obd)
+{
+        ENTRY;
+        if (OBT(obd) && OBP(obd, getref)) {
+                OBD_COUNTER_INCREMENT(obd, getref);
+                OBP(obd, getref)(obd);
+        }
+        EXIT;
+}
+
+static inline void obd_putref(struct obd_device *obd)
+{
+        ENTRY;
+        if (OBT(obd) && OBP(obd, putref)) {
+                OBD_COUNTER_INCREMENT(obd, putref);
+                OBP(obd, putref)(obd);
+        }
+        EXIT;
+}
+
 static inline int obd_init_export(struct obd_export *exp)
 {
         int rc = 0;
diff --git a/lustre/lov/lov_dev.c b/lustre/lov/lov_dev.c
index fd5e24a..6aa4431 100644
--- a/lustre/lov/lov_dev.c
+++ b/lustre/lov/lov_dev.c
@@ -421,7 +421,7 @@ static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev,
         int rc;
         ENTRY;
 
-        lov_getref(obd);
+        obd_getref(obd);
 
         tgt = obd->u.lov.lov_tgts[index];
         LASSERT(tgt != NULL);
@@ -450,7 +450,7 @@ static int lov_cl_add_target(const struct lu_env *env, struct lu_device *dev,
                         rc = PTR_ERR(cl);
                 }
         }
-        lov_putref(obd);
+        obd_putref(obd);
         RETURN(rc);
 }
 
@@ -463,7 +463,7 @@ static int lov_process_config(const struct lu_env *env,
         int gen;
         __u32 index;
 
-        lov_getref(obd);
+        obd_getref(obd);
 
         cmd = cfg->lcfg_command;
         rc = lov_process_config_base(d->ld_obd, cfg, &index, &gen);
@@ -480,7 +480,7 @@ static int lov_process_config(const struct lu_env *env,
                         break;
                 }
         }
-        lov_putref(obd);
+        obd_putref(obd);
         RETURN(rc);
 }
 
diff --git a/lustre/lov/lov_internal.h b/lustre/lov/lov_internal.h
index 69f6848..37015e2 100644
--- a/lustre/lov/lov_internal.h
+++ b/lustre/lov/lov_internal.h
@@ -37,6 +37,7 @@
 #ifndef LOV_INTERNAL_H
 #define LOV_INTERNAL_H
 
+#include <obd_class.h>
 #include <lustre/lustre_user.h>
 
 struct lov_lock_handles {
@@ -247,8 +248,6 @@ void lov_fix_desc_stripe_count(__u32 *val);
 void lov_fix_desc_pattern(__u32 *val);
 void lov_fix_desc_qos_maxage(__u32 *val);
 int lov_get_stripecnt(struct lov_obd *lov, __u32 stripe_count);
-void lov_getref(struct obd_device *obd);
-void lov_putref(struct obd_device *obd);
 int lov_connect_obd(struct obd_device *obd, __u32 index, int activate,
                     struct obd_connect_data *data);
 int lov_setup(struct obd_device *obd, struct lustre_cfg *lcfg);
diff --git a/lustre/lov/lov_log.c b/lustre/lov/lov_log.c
index 9e4fc79..81a0e00 100644
--- a/lustre/lov/lov_log.c
+++ b/lustre/lov/lov_log.c
@@ -132,7 +132,7 @@ static int lov_llog_origin_connect(struct llog_ctxt *ctxt,
         int i, rc = 0, err = 0;
         ENTRY;
 
-        lov_getref(obd);
+        obd_getref(obd);
         for (i = 0; i < lov->desc.ld_tgt_count; i++) {
                 struct obd_device *child;
                 struct llog_ctxt *cctxt;
@@ -153,7 +153,7 @@ static int lov_llog_origin_connect(struct llog_ctxt *ctxt,
                                 err = rc;
                 }
         }
-        lov_putref(obd);
+        obd_putref(obd);
 
         RETURN(err);
 }
@@ -171,7 +171,7 @@ static int lov_llog_repl_cancel(struct llog_ctxt *ctxt, struct lov_stripe_md *ls
         LASSERT(count == lsm->lsm_stripe_count);
 
         lov = &obd->u.lov;
-        lov_getref(obd);
+        obd_getref(obd);
         for (i = 0; i < count; i++, cookies++) {
                 struct lov_oinfo *loi = lsm->lsm_oinfo[i];
                 struct obd_device *child =
@@ -190,7 +190,7 @@ static int lov_llog_repl_cancel(struct llog_ctxt *ctxt, struct lov_stripe_md *ls
                                 rc = err;
                 }
         }
-        lov_putref(obd);
+        obd_putref(obd);
         RETURN(rc);
 }
 
@@ -226,7 +226,7 @@ int lov_llog_init(struct obd_device *obd, struct obd_llog_group *olg,
         if (rc)
                 GOTO(err_cleanup, rc);
 
-        lov_getref(obd);
+        obd_getref(obd);
         /* count may not match lov->desc.ld_tgt_count during dynamic ost add */
         for (i = 0; i < lov->desc.ld_tgt_count; i++) {
                 if (!lov->lov_tgts[i] || !lov->lov_tgts[i]->ltd_active)
@@ -243,7 +243,7 @@ int lov_llog_init(struct obd_device *obd, struct obd_llog_group *olg,
                                rc);
                 break;
         }
-        lov_putref(obd);
+        obd_putref(obd);
         GOTO(err_cleanup, rc);
 err_cleanup:
         if (rc) {
diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c
index 49dee08..fd6ba09 100644
--- a/lustre/lov/lov_obd.c
+++ b/lustre/lov/lov_obd.c
@@ -68,10 +68,9 @@
 
 #include "lov_internal.h"
 
-
 /* Keep a refcount of lov->tgt usage to prevent racing with addition/deletion.
    Any function that expects lov_tgts to remain stationary must take a ref. */
-void lov_getref(struct obd_device *obd)
+static void lov_getref(struct obd_device *obd)
 {
         struct lov_obd *lov = &obd->u.lov;
 
@@ -84,7 +83,7 @@ void lov_getref(struct obd_device *obd)
 
 static void __lov_del_obd(struct obd_device *obd, struct lov_tgt_desc *tgt);
 
-void lov_putref(struct obd_device *obd)
+static void lov_putref(struct obd_device *obd)
 {
         struct lov_obd *lov = &obd->u.lov;
 
@@ -258,7 +257,7 @@ static int lov_connect(const struct lu_env *env,
         if (data)
                 lov->lov_ocd = *data;
 
-        lov_getref(obd);
+        obd_getref(obd);
         for (i = 0; i < lov->desc.ld_tgt_count; i++) {
                 tgt = lov->lov_tgts[i];
                 if (!tgt || obd_uuid_empty(&tgt->ltd_uuid))
@@ -281,7 +280,7 @@ static int lov_connect(const struct lu_env *env,
                                obd->obd_name, rc);
                 }
         }
-        lov_putref(obd);
+        obd_putref(obd);
 
         RETURN(0);
 }
@@ -363,7 +362,7 @@ static int lov_disconnect(struct obd_export *exp)
 
         /* Let's hold another reference so lov_del_obd doesn't spin through
            putref every time */
-        lov_getref(obd);
+        obd_getref(obd);
 
         for (i = 0; i < lov->desc.ld_tgt_count; i++) {
                 if (lov->lov_tgts[i] && lov->lov_tgts[i]->ltd_exp) {
@@ -371,7 +370,7 @@ static int lov_disconnect(struct obd_export *exp)
                         lov_del_target(obd, i, 0, lov->lov_tgts[i]->ltd_gen);
                 }
         }
-        lov_putref(obd);
+        obd_putref(obd);
 
 out:
         rc = class_disconnect(exp); /* bz 9811 */
@@ -396,7 +395,7 @@ static int lov_set_osc_active(struct obd_device *obd, struct obd_uuid *uuid,
         CDEBUG(D_INFO, "Searching in lov %p for uuid %s (activate=%d)\n",
                lov, uuid->uuid, activate);
 
-        lov_getref(obd);
+        obd_getref(obd);
         for (index = 0; index < lov->desc.ld_tgt_count; index++) {
                 tgt = lov->lov_tgts[index];
                 if (!tgt || !tgt->ltd_exp)
@@ -434,7 +433,7 @@ static int lov_set_osc_active(struct obd_device *obd, struct obd_uuid *uuid,
         lov->lov_tgts[index]->ltd_qos.ltq_penalty = 0;
 
  out:
-        lov_putref(obd);
+        obd_putref(obd);
         RETURN(index);
 }
 
@@ -480,7 +479,7 @@ static int lov_notify(struct obd_device *obd, struct obd_device *watched,
                 struct lov_obd *lov = &obd->u.lov;
                 struct obd_device *tgt_obd;
                 int i;
-                lov_getref(obd);
+                obd_getref(obd);
                 for (i = 0; i < lov->desc.ld_tgt_count; i++) {
                         /* don't send sync event if target not
                          * connected/activated */
@@ -503,7 +502,7 @@ static int lov_notify(struct obd_device *obd, struct obd_device *watched,
                                 break;
                         }
                 }
-                lov_putref(obd);
+                obd_putref(obd);
         }
 
         RETURN(rc);
@@ -608,7 +607,7 @@ int lov_add_target(struct obd_device *obd, struct obd_uuid *uuidp,
                 RETURN(0);
         }
 
-        lov_getref(obd);
+        obd_getref(obd);
 
         rc = lov_connect_obd(obd, index, active, &lov->lov_ocd);
         if (rc)
@@ -628,7 +627,7 @@ out:
                        obd_uuid2str(&tgt->ltd_uuid));
                 lov_del_target(obd, index, 0, 0);
         }
-        lov_putref(obd);
+        obd_putref(obd);
         RETURN(rc);
 }
 
@@ -647,7 +646,7 @@ int lov_del_target(struct obd_device *obd, __u32 index,
                 RETURN(-EINVAL);
         }
 
-        lov_getref(obd);
+        obd_getref(obd);
 
         if (!lov->lov_tgts[index]) {
                 CERROR("LOV target at index %d is not setup.\n", index);
@@ -668,9 +667,9 @@ int lov_del_target(struct obd_device *obd, __u32 index,
 
         lov->lov_tgts[index]->ltd_reap = 1;
         lov->lov_death_row++;
-        /* we really delete it from lov_putref */
+        /* we really delete it from obd_putref */
 out:
-        lov_putref(obd);
+        obd_putref(obd);
 
         RETURN(rc);
 }
@@ -892,7 +891,7 @@ static int lov_cleanup(struct obd_device *obd)
 
         if (lov->lov_tgts) {
                 int i;
-                lov_getref(obd);
+                obd_getref(obd);
                 for (i = 0; i < lov->desc.ld_tgt_count; i++) {
                         if (!lov->lov_tgts[i])
                                 continue;
@@ -909,7 +908,7 @@ static int lov_cleanup(struct obd_device *obd)
                                        atomic_read(&lov->lov_refcount));
                         lov_del_target(obd, i, 0, 0);
                 }
-                lov_putref(obd);
+                obd_putref(obd);
                 OBD_FREE(lov->lov_tgts, sizeof(*lov->lov_tgts) *
                          lov->lov_tgt_size);
                 lov->lov_tgt_size = 0;
@@ -1016,7 +1015,7 @@ static int lov_clear_orphans(struct obd_export *export, struct obdo *src_oa,
                        ost_uuid->uuid);
         }
 
-        lov_getref(export->exp_obd);
+        obd_getref(export->exp_obd);
         for (i = 0; i < lov->desc.ld_tgt_count; i++) {
                 struct lov_stripe_md obj_md;
                 struct lov_stripe_md *obj_mdp = &obj_md;
@@ -1057,7 +1056,7 @@ static int lov_clear_orphans(struct obd_export *export, struct obdo *src_oa,
                 if (ost_uuid)
                         break;
         }
-        lov_putref(export->exp_obd);
+        obd_putref(export->exp_obd);
 
         OBDO_FREE(tmp_oa);
         RETURN(rc);
@@ -1128,7 +1127,7 @@ static int lov_create(struct obd_export *exp, struct obdo *src_oa,
         if (!lov->desc.ld_active_tgt_count)
                 RETURN(-EIO);
 
-        lov_getref(exp->exp_obd);
+        obd_getref(exp->exp_obd);
         /* Recreate a specific object id at the given OST index */
         if ((src_oa->o_valid & OBD_MD_FLFLAGS) &&
             (src_oa->o_flags & OBD_FL_RECREATE_OBJS)) {
@@ -1154,7 +1153,7 @@ static int lov_create(struct obd_export *exp, struct obdo *src_oa,
         }
         rc = lov_fini_create_set(set, ea);
 out:
-        lov_putref(exp->exp_obd);
+        obd_putref(exp->exp_obd);
         RETURN(rc);
 }
 
@@ -1190,7 +1189,7 @@ static int lov_destroy(struct obd_export *exp, struct obdo *oa,
         }
 
         lov = &exp->exp_obd->u.lov;
-        lov_getref(exp->exp_obd);
+        obd_getref(exp->exp_obd);
         rc = lov_prep_destroy_set(exp, &oinfo, oa, lsm, oti, &set);
         if (rc)
                 GOTO(out, rc);
@@ -1220,7 +1219,7 @@ static int lov_destroy(struct obd_export *exp, struct obdo *oa,
         }
         err = lov_fini_destroy_set(set);
 out:
-        lov_putref(exp->exp_obd);
+        obd_putref(exp->exp_obd);
         RETURN(rc ? rc : err);
 }
 
@@ -2484,7 +2483,7 @@ static int lov_get_info(struct obd_export *exp, __u32 keylen,
         if (!vallen || !val)
                 RETURN(-EFAULT);
 
-        lov_getref(obddev);
+        obd_getref(obddev);
 
         if (KEY_IS(KEY_LOCK_TO_STRIPE)) {
                 struct {
@@ -2545,7 +2544,7 @@ static int lov_get_info(struct obd_export *exp, __u32 keylen,
         rc = -EINVAL;
 
 out:
-        lov_putref(obddev);
+        obd_putref(obddev);
         RETURN(rc);
 }
 
@@ -2571,7 +2570,7 @@ static int lov_set_info_async(struct obd_export *exp, obd_count keylen,
                         RETURN(-ENOMEM);
         }
 
-        lov_getref(obddev);
+        obd_getref(obddev);
         count = lov->desc.ld_tgt_count;
 
         if (KEY_IS(KEY_NEXT_ID)) {
@@ -2636,7 +2635,7 @@ static int lov_set_info_async(struct obd_export *exp, obd_count keylen,
                         rc = err;
         }
 
-        lov_putref(obddev);
+        obd_putref(obddev);
         if (no_set) {
                 err = ptlrpc_set_wait(set);
                 if (!rc)
@@ -2758,6 +2757,8 @@ struct obd_ops lov_obd_ops = {
         .o_pool_rem            = lov_pool_remove,
         .o_pool_add            = lov_pool_add,
         .o_pool_del            = lov_pool_del,
+        .o_getref              = lov_getref,
+        .o_putref              = lov_putref,
 };
 
 static quota_interface_t *quota_interface;
diff --git a/lustre/lov/lov_pool.c b/lustre/lov/lov_pool.c
index 3df37a7..fd41f84 100644
--- a/lustre/lov/lov_pool.c
+++ b/lustre/lov/lov_pool.c
@@ -550,7 +550,7 @@ int lov_pool_add(struct obd_device *obd, char *poolname, char *ostname)
 
 
         /* search ost in lov array */
-        lov_getref(obd);
+        obd_getref(obd);
         for (lov_idx = 0; lov_idx < lov->desc.ld_tgt_count; lov_idx++) {
                 if (!lov->lov_tgts[lov_idx])
                         continue;
@@ -573,7 +573,7 @@ int lov_pool_add(struct obd_device *obd, char *poolname, char *ostname)
 
         EXIT;
 out:
-        lov_putref(obd);
+        obd_putref(obd);
         lov_pool_putref(pool);
         return rc;
 }
@@ -595,7 +595,7 @@ int lov_pool_remove(struct obd_device *obd, char *poolname, char *ostname)
 
         obd_str2uuid(&ost_uuid, ostname);
 
-        lov_getref(obd);
+        obd_getref(obd);
         /* search ost in lov array, to get index */
         for (lov_idx = 0; lov_idx < lov->desc.ld_tgt_count; lov_idx++) {
                 if (!lov->lov_tgts[lov_idx])
@@ -619,7 +619,7 @@ int lov_pool_remove(struct obd_device *obd, char *poolname, char *ostname)
 
         EXIT;
 out:
-        lov_putref(obd);
+        obd_putref(obd);
         lov_pool_putref(pool);
         return rc;
 }
diff --git a/lustre/lov/lov_qos.c b/lustre/lov/lov_qos.c
index 7842f8f..add23ae 100644
--- a/lustre/lov/lov_qos.c
+++ b/lustre/lov/lov_qos.c
@@ -764,7 +764,7 @@ static int alloc_qos(struct obd_export *exp, int *idx_arr, int *stripe_cnt,
                 lqr = &(pool->pool_rr);
         }
 
-        lov_getref(exp->exp_obd);
+        obd_getref(exp->exp_obd);
 
         /* wait for fresh statfs info if needed, the rpcs are sent in
          * lov_create() */
@@ -933,7 +933,7 @@ out_nolock:
         if (rc == -EAGAIN)
                 rc = alloc_rr(lov, idx_arr, stripe_cnt, poolname, flags);
 
-        lov_putref(exp->exp_obd);
+        obd_putref(exp->exp_obd);
         RETURN(rc);
 }
 
diff --git a/lustre/mdt/mdt_handler.c b/lustre/mdt/mdt_handler.c
index d7aaaf9..5d639f4 100644
--- a/lustre/mdt/mdt_handler.c
+++ b/lustre/mdt/mdt_handler.c
@@ -5292,7 +5292,8 @@ static int mdt_upcall(const struct lu_env *env, struct md_device *md,
                         break;
 #ifdef HAVE_QUOTA_SUPPORT
                 case MD_LOV_QUOTA:
-                        if (md->md_lu_dev.ld_obd->obd_recovering == 0)
+                        if (md->md_lu_dev.ld_obd->obd_recovering == 0 &&
+                            likely(md->md_lu_dev.ld_obd->obd_stopping == 0))
                                 next->md_ops->mdo_quota.mqo_recovery(env, next);
                         break;
 #endif
@@ -5493,7 +5494,8 @@ int mdt_postrecov(const struct lu_env *env, struct mdt_device *mdt)
 
         rc = ld->ld_ops->ldo_recovery_complete(env, ld);
 #ifdef HAVE_QUOTA_SUPPORT
-        next->md_ops->mdo_quota.mqo_recovery(env, next);
+        if (likely(obd->obd_stopping == 0))
+                next->md_ops->mdo_quota.mqo_recovery(env, next);
 #endif
         RETURN(rc);
 }
diff --git a/lustre/obdclass/lprocfs_status.c b/lustre/obdclass/lprocfs_status.c
index 9ce1cb4..f50c739 100644
--- a/lustre/obdclass/lprocfs_status.c
+++ b/lustre/obdclass/lprocfs_status.c
@@ -1417,6 +1417,8 @@ void lprocfs_init_ops_stats(int num_private_stats, struct lprocfs_stats *stats)
         LPROCFS_OBD_OP_INIT(num_private_stats, stats, pool_rem);
         LPROCFS_OBD_OP_INIT(num_private_stats, stats, pool_add);
         LPROCFS_OBD_OP_INIT(num_private_stats, stats, pool_del);
+        LPROCFS_OBD_OP_INIT(num_private_stats, stats, getref);
+        LPROCFS_OBD_OP_INIT(num_private_stats, stats, putref);
 }
 
 int lprocfs_alloc_obd_stats(struct obd_device *obd, unsigned num_private_stats)
diff --git a/lustre/quota/quota_ctl.c b/lustre/quota/quota_ctl.c
index be5bab6..8a3d2e8 100644
--- a/lustre/quota/quota_ctl.c
+++ b/lustre/quota/quota_ctl.c
@@ -352,6 +352,7 @@ int lov_quota_ctl(struct obd_device *unused, struct obd_export *exp,
                 RETURN(-EFAULT);
         }
 
+        obd_getref(obd);
         for (i = 0; i < lov->desc.ld_tgt_count; i++) {
                 int err;
 
@@ -377,6 +378,7 @@ int lov_quota_ctl(struct obd_device *unused, struct obd_export *exp,
                         bhardlimit += oqctl->qc_dqblk.dqb_bhardlimit;
                 }
         }
+        obd_putref(obd);
 
         if (oqctl->qc_cmd == Q_GETOQUOTA) {
                 oqctl->qc_dqblk.dqb_curspace = curspace;
diff --git a/lustre/quota/quota_master.c b/lustre/quota/quota_master.c
index ab5214f..d7873da 100644
--- a/lustre/quota/quota_master.c
+++ b/lustre/quota/quota_master.c
@@ -1553,17 +1553,18 @@ static int qmaster_recovery_main(void *arg)
 {
         struct qmaster_recov_thread_data *data = arg;
         struct obd_device *obd = data->obd;
+        struct mds_obd *mds = &obd->u.mds;
+        struct lustre_quota_info *qinfo = &mds->mds_quota_info;
         int rc = 0;
         unsigned short type;
         ENTRY;
 
         ptlrpc_daemonize("qmaster_recovd");
 
+        class_incref(obd, "qmaster_recovd", obd);
         complete(&data->comp);
 
         for (type = USRQUOTA; type < MAXQUOTAS; type++) {
-                struct mds_obd *mds = &obd->u.mds;
-                struct lustre_quota_info *qinfo = &mds->mds_quota_info;
                 struct list_head id_list;
                 struct dquot_id *dqid, *tmp;
 
@@ -1593,6 +1594,7 @@ free:
                         kfree(dqid);
                 }
         }
+        class_decref(obd, "qmaster_recovd", obd);
         RETURN(rc);
 }
 
-- 
1.8.3.1