LU-2258 mount: solve races between evict and umount

There are two race conditions between evict and umount.

1) The one is that obd_export_evict_by_xxx() can touch a lustre hash
which has already been finalized but not unregistered yet.

2) The other is that the class_decref() in lprocfs_wr_evict_client()
can call lprocfs_remove() via osc_cleanup() while having _lprocfs_lock
already. That's why this case ends up in a deadlock.

This patch solve these problems with the below solutions.
- see if obd_nid_hash and obd_uuid_hash is still available when
  evicting.
- move class_incref() to below LPROCFS_EXIT() and class_decref()
  before LPROCFS_ENTRY() in order to avoid dead-locking

Signed-off-by: Hiroya Nozaki <nozaki.hiroya@jp.fujitsu.com>
Change-Id: I2cc19a88ffd6a230ab115bc1e4b9d31fbbbb4615
Reviewed-on: http://review.whamcloud.com/4444
Tested-by: Hudson
Reviewed-by: Lai Siyao <laisiyao@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Jinshan Xiong <jinshan.xiong@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/obdclass/genops.c b/lustre/obdclass/genops.c
index 316c9f9..602abb8 100644 (file)
--- a/lustre/obdclass/genops.c
+++ b/lustre/obdclass/genops.c
@@ -1387,13 +1387,25 @@ EXPORT_SYMBOL(obd_export_nid2str);
 
 int obd_export_evict_by_nid(struct obd_device *obd, const char *nid)
 {
 
 int obd_export_evict_by_nid(struct obd_device *obd, const char *nid)
 {

+       cfs_hash_t *nid_hash;

         struct obd_export *doomed_exp = NULL;
         int exports_evicted = 0;
 
         lnet_nid_t nid_key = libcfs_str2nid((char *)nid);
 
         struct obd_export *doomed_exp = NULL;
         int exports_evicted = 0;
 
         lnet_nid_t nid_key = libcfs_str2nid((char *)nid);
 

+       cfs_spin_lock(&obd->obd_dev_lock);
+       /* umount has run already, so evict thread should leave
+        * its task to umount thread now */
+       if (obd->obd_stopping) {
+               cfs_spin_unlock(&obd->obd_dev_lock);
+               return exports_evicted;
+       }
+       nid_hash = obd->obd_nid_hash;
+       cfs_hash_getref(nid_hash);
+       cfs_spin_unlock(&obd->obd_dev_lock);
+

         do {
         do {

-                doomed_exp = cfs_hash_lookup(obd->obd_nid_hash, &nid_key);
+               doomed_exp = cfs_hash_lookup(nid_hash, &nid_key);

                 if (doomed_exp == NULL)
                         break;
 
                 if (doomed_exp == NULL)
                         break;
 


@@ -1411,6 +1423,8 @@ int obd_export_evict_by_nid(struct obd_device *obd, const char *nid)
                 class_export_put(doomed_exp);
         } while (1);
 
                 class_export_put(doomed_exp);
         } while (1);
 

+       cfs_hash_putref(nid_hash);
+

         if (!exports_evicted)
                 CDEBUG(D_HA,"%s: can't disconnect NID '%s': no exports found\n",
                        obd->obd_name, nid);
         if (!exports_evicted)
                 CDEBUG(D_HA,"%s: can't disconnect NID '%s': no exports found\n",
                        obd->obd_name, nid);


@@ -1420,17 +1434,28 @@ EXPORT_SYMBOL(obd_export_evict_by_nid);
 
 int obd_export_evict_by_uuid(struct obd_device *obd, const char *uuid)
 {
 
 int obd_export_evict_by_uuid(struct obd_device *obd, const char *uuid)
 {

+       cfs_hash_t *uuid_hash;

         struct obd_export *doomed_exp = NULL;
         struct obd_uuid doomed_uuid;
         int exports_evicted = 0;
 
         struct obd_export *doomed_exp = NULL;
         struct obd_uuid doomed_uuid;
         int exports_evicted = 0;
 

+       cfs_spin_lock(&obd->obd_dev_lock);
+       if (obd->obd_stopping) {
+               cfs_spin_unlock(&obd->obd_dev_lock);
+               return exports_evicted;
+       }
+       uuid_hash = obd->obd_uuid_hash;
+       cfs_hash_getref(uuid_hash);
+       cfs_spin_unlock(&obd->obd_dev_lock);
+

         obd_str2uuid(&doomed_uuid, uuid);
         if (obd_uuid_equals(&doomed_uuid, &obd->obd_uuid)) {
                 CERROR("%s: can't evict myself\n", obd->obd_name);
         obd_str2uuid(&doomed_uuid, uuid);
         if (obd_uuid_equals(&doomed_uuid, &obd->obd_uuid)) {
                 CERROR("%s: can't evict myself\n", obd->obd_name);

+               cfs_hash_putref(uuid_hash);

                 return exports_evicted;
         }
 
                 return exports_evicted;
         }
 

-        doomed_exp = cfs_hash_lookup(obd->obd_uuid_hash, &doomed_uuid);
+       doomed_exp = cfs_hash_lookup(uuid_hash, &doomed_uuid);

 
         if (doomed_exp == NULL) {
                 CERROR("%s: can't disconnect %s: no exports found\n",
 
         if (doomed_exp == NULL) {
                 CERROR("%s: can't disconnect %s: no exports found\n",


@@ -1442,6 +1467,7 @@ int obd_export_evict_by_uuid(struct obd_device *obd, const char *uuid)
                 class_export_put(doomed_exp);
                 exports_evicted++;
         }
                 class_export_put(doomed_exp);
                 exports_evicted++;
         }

+       cfs_hash_putref(uuid_hash);

 
         return exports_evicted;
 }
 
         return exports_evicted;
 }

diff --git a/lustre/ptlrpc/lproc_ptlrpc.c b/lustre/ptlrpc/lproc_ptlrpc.c
index 0b87bc9..0caad4a 100644 (file)
--- a/lustre/ptlrpc/lproc_ptlrpc.c
+++ b/lustre/ptlrpc/lproc_ptlrpc.c
@@ -866,8 +866,8 @@ int lprocfs_wr_evict_client(struct file *file, const char *buffer,
          * the proc entries under the being destroyed export{}, so I have
          * to drop the lock at first here.
          * - jay, jxiong@clusterfs.com */
          * the proc entries under the being destroyed export{}, so I have
          * to drop the lock at first here.
          * - jay, jxiong@clusterfs.com */

-        class_incref(obd, __FUNCTION__, cfs_current());

         LPROCFS_EXIT();
         LPROCFS_EXIT();

+       class_incref(obd, __FUNCTION__, cfs_current());

 
         if (strncmp(tmpbuf, "nid:", 4) == 0)
                 obd_export_evict_by_nid(obd, tmpbuf + 4);
 
         if (strncmp(tmpbuf, "nid:", 4) == 0)
                 obd_export_evict_by_nid(obd, tmpbuf + 4);


@@ -876,8 +876,8 @@ int lprocfs_wr_evict_client(struct file *file, const char *buffer,
         else
                 obd_export_evict_by_uuid(obd, tmpbuf);
 
         else
                 obd_export_evict_by_uuid(obd, tmpbuf);
 

+       class_decref(obd, __FUNCTION__, cfs_current());

         LPROCFS_ENTRY();
         LPROCFS_ENTRY();

-        class_decref(obd, __FUNCTION__, cfs_current());

 
 out:
         OBD_FREE(kbuf, BUFLEN);
 
 out:
         OBD_FREE(kbuf, BUFLEN);