don't call obd_disconnect under lov_lock.

Branch HEAD
b=17310
i=johann
i=rread

diff --git a/lustre/ChangeLog b/lustre/ChangeLog
index 2d6cf88..171283c 100644 (file)
--- a/lustre/ChangeLog
+++ b/lustre/ChangeLog
@@ -14,6 +14,13 @@ tbd  Sun Microsystems, Inc.
        * File join has been disabled in this release, refer to Bugzilla 16929.
 
 Severity   : normal
+Frequency  : rare, connect and disconnect target at same time
+Bugzilla   : 17310
+Descriptoin: ASSERTION(atomic_read(&imp->imp_inflight) == 0
+Details    : don't call obd_disconnect under lov_lock. this long time
+             operation and can block ptlrpcd which answer to connect request.
+
+Severity   : normal
 Frequency  : rare
 Bugzilla   : 18154
 Descriptoin: don't lose wakeup for imp_recovery_waitq

diff --git a/lustre/include/obd.h b/lustre/include/obd.h
index 941bae3..a351c93 100644 (file)
--- a/lustre/include/obd.h
+++ b/lustre/include/obd.h
@@ -648,6 +648,7 @@ struct lov_qos {
 };
 
 struct lov_tgt_desc {
+        struct list_head    ltd_kill;
         struct obd_uuid     ltd_uuid;
         struct obd_export  *ltd_exp;
         struct ltd_qos      ltd_qos;     /* qos info per target */

diff --git a/lustre/lov/lov_internal.h b/lustre/lov/lov_internal.h
index c6c3a69..b3a763a 100644 (file)
--- a/lustre/lov/lov_internal.h
+++ b/lustre/lov/lov_internal.h
@@ -162,7 +162,7 @@ int lov_stripe_number(struct lov_stripe_md *lsm, obd_off lov_off);
 #define LOV_USES_ASSIGNED_STRIPE        0
 #define LOV_USES_DEFAULT_STRIPE         1
 int qos_add_tgt(struct obd_device *obd, __u32 index);
-int qos_del_tgt(struct obd_device *obd, __u32 index);
+int qos_del_tgt(struct obd_device *obd, struct lov_tgt_desc *tgt);
 void qos_shrink_lsm(struct lov_request_set *set);
 int qos_prep_create(struct obd_export *exp, struct lov_request_set *set);
 void qos_update(struct lov_obd *lov);

diff --git a/lustre/lov/lov_obd.c b/lustre/lov/lov_obd.c
index 86058ed..4fa94d2 100644 (file)
--- a/lustre/lov/lov_obd.c
+++ b/lustre/lov/lov_obd.c
@@ -82,26 +82,43 @@ void lov_getref(struct obd_device *obd)
         return;
 }
 
-static void __lov_del_obd(struct obd_device *obd, __u32 index);
+static void __lov_del_obd(struct obd_device *obd, struct lov_tgt_desc *tgt);
 
 void lov_putref(struct obd_device *obd)
 {
         struct lov_obd *lov = &obd->u.lov;
+
         mutex_down(&lov->lov_lock);
         /* ok to dec to 0 more than once -- ltd_exp's will be null */
         if (atomic_dec_and_test(&lov->lov_refcount) && lov->lov_death_row) {
+                CFS_LIST_HEAD(kill);
                 int i;
+                struct lov_tgt_desc *tgt, *n;
                 CDEBUG(D_CONFIG, "destroying %d lov targets\n",
                        lov->lov_death_row);
                 for (i = 0; i < lov->desc.ld_tgt_count; i++) {
-                        if (!lov->lov_tgts[i] || !lov->lov_tgts[i]->ltd_reap)
+                        tgt = lov->lov_tgts[i];
+
+                        if (!tgt || !tgt->ltd_reap)
                                 continue;
-                        /* Disconnect and delete from list */
-                        __lov_del_obd(obd, i);
+                        list_add(&tgt->ltd_kill, &kill);
+                        /* XXX - right now there is a dependency on ld_tgt_count
+                         * being the maximum tgt index for computing the
+                         * mds_max_easize. So we can't shrink it. */
+                        lov_ost_pool_remove(&lov->lov_packed, i);
+                        lov->lov_tgts[i] = NULL;
                         lov->lov_death_row--;
                 }
+                mutex_up(&lov->lov_lock);
+
+                list_for_each_entry_safe(tgt, n, &kill, ltd_kill) {
+                        list_del(&tgt->ltd_kill);
+                        /* Disconnect */
+                        __lov_del_obd(obd, tgt);
+                }
+        } else {
+                mutex_up(&lov->lov_lock);
         }
-        mutex_up(&lov->lov_lock);
 }
 
 static int lov_set_osc_active(struct obd_device *obd, struct obd_uuid *uuid,
@@ -288,26 +305,22 @@ static int lov_connect(const struct lu_env *env,
         RETURN(0);
 }
 
-static int lov_disconnect_obd(struct obd_device *obd, __u32 index)
+static int lov_disconnect_obd(struct obd_device *obd, struct lov_tgt_desc *tgt)
 {
         cfs_proc_dir_entry_t *lov_proc_dir;
         struct lov_obd *lov = &obd->u.lov;
         struct obd_device *osc_obd;
         int rc;
-
         ENTRY;
 
-        if (lov->lov_tgts[index] == NULL)
-                RETURN(-EINVAL);
-
-        osc_obd = class_exp2obd(lov->lov_tgts[index]->ltd_exp);
+        osc_obd = class_exp2obd(tgt->ltd_exp);
         CDEBUG(D_CONFIG, "%s: disconnecting target %s\n",
                obd->obd_name, osc_obd->obd_name);
 
-        if (lov->lov_tgts[index]->ltd_active) {
-                lov->lov_tgts[index]->ltd_active = 0;
+        if (tgt->ltd_active) {
+                tgt->ltd_active = 0;
                 lov->desc.ld_active_tgt_count--;
-                lov->lov_tgts[index]->ltd_exp->exp_obd->obd_inactive = 1;
+                tgt->ltd_exp->exp_obd->obd_inactive = 1;
         }
 
         lov_proc_dir = lprocfs_srch(obd->obd_proc_entry, "target_obds");
@@ -336,16 +349,16 @@ static int lov_disconnect_obd(struct obd_device *obd, __u32 index)
 
         obd_register_observer(osc_obd, NULL);
 
-        rc = obd_disconnect(lov->lov_tgts[index]->ltd_exp);
+        rc = obd_disconnect(tgt->ltd_exp);
         if (rc) {
                 CERROR("Target %s disconnect error %d\n",
-                       lov_uuid2str(lov, index), rc);
+                       tgt->ltd_uuid.uuid, rc);
                 rc = 0;
         }
 
-        qos_del_tgt(obd, index);
+        qos_del_tgt(obd, tgt);
 
-        lov->lov_tgts[index]->ltd_exp = NULL;
+        tgt->ltd_exp = NULL;
         RETURN(0);
 }
 
@@ -671,12 +684,9 @@ out:
         RETURN(rc);
 }
 
-/* We are holding lov_lock */
-static void __lov_del_obd(struct obd_device *obd, __u32 index)
+static void __lov_del_obd(struct obd_device *obd, struct lov_tgt_desc *tgt)
 {
-        struct lov_obd *lov = &obd->u.lov;
         struct obd_device *osc_obd;
-        struct lov_tgt_desc *tgt = lov->lov_tgts[index];
 
         LASSERT(tgt);
         LASSERT(tgt->ltd_reap);
@@ -684,18 +694,12 @@ static void __lov_del_obd(struct obd_device *obd, __u32 index)
         osc_obd = class_exp2obd(tgt->ltd_exp);
 
         CDEBUG(D_CONFIG, "Removing tgt %s : %s\n",
-               lov_uuid2str(lov, index),
+               tgt->ltd_uuid.uuid,
                osc_obd ? osc_obd->obd_name : "<no obd>");
 
         if (tgt->ltd_exp)
-                lov_disconnect_obd(obd, index);
-
-        /* XXX - right now there is a dependency on ld_tgt_count being the
-         * maximum tgt index for computing the mds_max_easize. So we can't
-         * shrink it. */
+                lov_disconnect_obd(obd, tgt);
 
-        lov_ost_pool_remove(&lov->lov_packed, index);
-        lov->lov_tgts[index] = NULL;
         OBD_FREE_PTR(tgt);
 
         /* Manual cleanup - no cleanup logs to clean up the osc's.  We must

diff --git a/lustre/lov/lov_qos.c b/lustre/lov/lov_qos.c
index 45245ed..d020dd7 100644 (file)
--- a/lustre/lov/lov_qos.c
+++ b/lustre/lov/lov_qos.c
@@ -121,19 +121,16 @@ out:
         RETURN(rc);
 }
 
-int qos_del_tgt(struct obd_device *obd, __u32 index)
+int qos_del_tgt(struct obd_device *obd, struct lov_tgt_desc *tgt)
 {
         struct lov_obd *lov = &obd->u.lov;
         struct lov_qos_oss *oss;
         int rc = 0;
         ENTRY;
 
-        if (!lov->lov_tgts[index])
-                RETURN(0);
-
         down_write(&lov->lov_qos.lq_rw_sem);
 
-        oss = lov->lov_tgts[index]->ltd_qos.ltq_oss;
+        oss = tgt->ltd_qos.ltq_oss;
         if (!oss)
                 GOTO(out, rc = -ENOENT);
 

diff --git a/lustre/ptlrpc/import.c b/lustre/ptlrpc/import.c
index 573f7a3..6a19364 100644 (file)
--- a/lustre/ptlrpc/import.c
+++ b/lustre/ptlrpc/import.c
@@ -318,16 +318,6 @@ void ptlrpc_invalidate_import(struct obd_import *imp)
                         }
 
                         if (atomic_read(&imp->imp_unregistering) == 0) {
-                                /* XXX: This is temporary workaround for long 
-                                 * connect interpret due to locking in lov in 
-                                 * in import activation path, which causes
-                                 * connect rpc stay on sending list longer
-                                 * time. Let's wait longer insread of asserting
-                                 * here.
-                                 *
-                                 * Activation should not be blocking. Kill
-                                 * this #if 0 when activation code is fixed. */
-#if 0
                                 /* We know that only "unregistering" rpcs may
                                  * still survive in sending or delaying lists
                                  * (They are waiting for long reply unlink in
@@ -340,7 +330,6 @@ void ptlrpc_invalidate_import(struct obd_import *imp)
                                  * dropped to zero. No new inflights possible at
                                  * this point. */
                                 rc = 0;
-#endif
                         } else {
                                 CERROR("%s: RPCs in \"%s\" phase found (%d). "
                                        "Network is sluggish? Waiting them "