LU-6040 lnet: remove messages from lazy portal on NI shutdown

When shutting down an NI in a busy system, some messages received
on this NI, might be on the lazy portal.  They would have grabbed
a ref count on the NI.  Therefore NI will not be removed until
messages are processed.

In order to avoid this scenario, when an NI is shutdown go through
all messages queued on the lazy portal and drop messages for the
NI being shutdown

Signed-off-by: Amir Shehata <amir.shehata@intel.com>
Change-Id: I67c8b720a6eb62fded4f084c1acea69dcdc8d2b6
Reviewed-on: http://review.whamcloud.com/13836
Tested-by: Jenkins
Tested-by: Maloo <hpdd-maloo@intel.com>
Reviewed-by: Isaac Huang <he.huang@intel.com>
Reviewed-by: Liang Zhen <liang.zhen@intel.com>
Reviewed-by: Oleg Drokin <oleg.drokin@intel.com>

diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h
index 3daa92c..db29f2d 100644 (file)
--- a/lnet/include/lnet/lib-lnet.h
+++ b/lnet/include/lnet/lib-lnet.h
@@ -491,6 +491,7 @@ int lnet_dyn_add_ni(lnet_pid_t requested_pid, char *nets,
                    __s32 peer_timeout, __s32 peer_cr, __s32 peer_buf_cr,
                    __s32 credits);
 int lnet_dyn_del_ni(__u32 net);
+int lnet_clear_lazy_portal(struct lnet_ni *ni, int portal, char *reason);
 
 int lnet_islocalnid(lnet_nid_t nid);
 int lnet_islocalnet(__u32 net);

diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c
index 99a1af7..270bb79 100644 (file)
--- a/lnet/lnet/api-ni.c
+++ b/lnet/lnet/api-ni.c
@@ -1204,10 +1204,16 @@ lnet_shutdown_lndnis(void)
 static void
 lnet_shutdown_lndni(struct lnet_ni *ni)
 {
+       int i;
+
        lnet_net_lock(LNET_LOCK_EX);
        lnet_ni_unlink_locked(ni);
        lnet_net_unlock(LNET_LOCK_EX);
 
+       /* clear messages for this NI on the lazy portal */
+       for (i = 0; i < the_lnet.ln_nportals; i++)
+               lnet_clear_lazy_portal(ni, i, "Shutting down NI");
+
        /* Do peer table cleanup for this ni */
        lnet_peer_tables_cleanup(ni);
 

diff --git a/lnet/lnet/lib-ptl.c b/lnet/lnet/lib-ptl.c
index 98fbeac..b55e993 100644 (file)
--- a/lnet/lnet/lib-ptl.c
+++ b/lnet/lnet/lib-ptl.c
@@ -889,20 +889,11 @@ LNetSetLazyPortal(int portal)
 }
 EXPORT_SYMBOL(LNetSetLazyPortal);
 
-/**
- * Turn off the lazy portal attribute. Delayed requests on the portal,
- * if any, will be all dropped when this function returns.
- *
- * \param portal Index of the portal to disable the lazy attribute on.
- *
- * \retval 0       On success.
- * \retval -EINVAL If \a portal is not a valid index.
- */
 int
-LNetClearLazyPortal(int portal)
+lnet_clear_lazy_portal(struct lnet_ni *ni, int portal, char *reason)
 {
        struct lnet_portal      *ptl;
-       struct list_head        zombies = LIST_HEAD_INIT(zombies);
+       struct list_head        zombies = LIST_HEAD_INIT(zombies);
 
        if (portal < 0 || portal >= the_lnet.ln_nportals)
                return -EINVAL;
@@ -918,21 +909,48 @@ LNetClearLazyPortal(int portal)
                return 0;
        }
 
-       if (the_lnet.ln_shutdown)
-               CWARN("Active lazy portal %d on exit\n", portal);
-       else
-               CDEBUG(D_NET, "clearing portal %d lazy\n", portal);
+       if (ni != NULL) {
+               struct lnet_msg *msg, *tmp;
 
-       /* grab all the blocked messages atomically */
-       list_splice_init(&ptl->ptl_msg_delayed, &zombies);
+               /* grab all messages which are on the NI passed in */
+               list_for_each_entry_safe(msg, tmp, &ptl->ptl_msg_delayed,
+                                        msg_list) {
+                       if (msg->msg_rxpeer->lp_ni == ni)
+                               list_move(&msg->msg_list, &zombies);
+               }
+       } else {
+               if (the_lnet.ln_shutdown)
+                       CWARN("Active lazy portal %d on exit\n", portal);
+               else
+                       CDEBUG(D_NET, "clearing portal %d lazy\n", portal);
+
+               /* grab all the blocked messages atomically */
+               list_splice_init(&ptl->ptl_msg_delayed, &zombies);
 
-       lnet_ptl_unsetopt(ptl, LNET_PTL_LAZY);
+               lnet_ptl_unsetopt(ptl, LNET_PTL_LAZY);
+       }
 
        lnet_ptl_unlock(ptl);
        lnet_res_unlock(LNET_LOCK_EX);
 
-       lnet_drop_delayed_msg_list(&zombies, "Clearing lazy portal attr");
+       lnet_drop_delayed_msg_list(&zombies, reason);
 
        return 0;
 }
+
+/**
+ * Turn off the lazy portal attribute. Delayed requests on the portal,
+ * if any, will be all dropped when this function returns.
+ *
+ * \param portal Index of the portal to disable the lazy attribute on.
+ *
+ * \retval 0       On success.
+ * \retval -EINVAL If \a portal is not a valid index.
+ */
+int
+LNetClearLazyPortal(int portal)
+{
+       return lnet_clear_lazy_portal(NULL, portal,
+                                     "Clearing lazy portal attr");
+}
 EXPORT_SYMBOL(LNetClearLazyPortal);