LU-1432 ptlrpc: LBUG in lprocfs_free_client_stats()

* serialize connect and target obd cleanup to avoid connect
  accessing unexisted data structure.
* connect export refcounting cleanup.

Signed-off-by: Hiroya Nozaki <nozaki.hiroya@jp.fujitsu.com>
Signed-off-by: Lai Siyao <laisiyao@whamcloud.com>
Change-Id: I0a9e8a58ecdc1212565a478f4a758755a1b95f99
Reviewed-on: http://review.whamcloud.com/3244
Tested-by: Hudson
Reviewed-by: Bobi Jam <bobijam@whamcloud.com>
Tested-by: Maloo <whamcloud.maloo@gmail.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lustre/include/obd.h b/lustre/include/obd.h
index 4272774..d5818a1 100644 (file)
--- a/lustre/include/obd.h
+++ b/lustre/include/obd.h
@@ -1181,6 +1181,8 @@ struct obd_device {
          * debugging.
          */
         struct lu_ref          obd_reference;
          * debugging.
          */
         struct lu_ref          obd_reference;

+
+       int                    obd_conn_inprogress;

 };
 
 #define OBD_LLOG_FL_SENDNOW     0x0001
 };
 
 #define OBD_LLOG_FL_SENDNOW     0x0001

diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c
index 25c56ef..19883b2 100644 (file)
--- a/lustre/ldlm/ldlm_lib.c
+++ b/lustre/ldlm/ldlm_lib.c
@@ -731,7 +731,7 @@ check_and_start_recovery_timer(struct obd_device *obd,
 
 int target_handle_connect(struct ptlrpc_request *req)
 {
 
 int target_handle_connect(struct ptlrpc_request *req)
 {

-        struct obd_device *target, *targref = NULL;
+       struct obd_device *target = NULL, *targref = NULL;

         struct obd_export *export = NULL;
         struct obd_import *revimp;
         struct lustre_handle conn;
         struct obd_export *export = NULL;
         struct obd_import *revimp;
         struct lustre_handle conn;


@@ -762,17 +762,29 @@ int target_handle_connect(struct ptlrpc_request *req)
         if (!target)
                 target = class_name2obd(str);
 
         if (!target)
                 target = class_name2obd(str);
 

-        if (!target || target->obd_stopping || !target->obd_set_up) {
-                deuuidify(str, NULL, &target_start, &target_len);
-                LCONSOLE_ERROR_MSG(0x137, "%.*s: Not available for connect "
-                                   "from %s (%s)\n", target_len, target_start,
-                                   libcfs_nid2str(req->rq_peer.nid), !target ?
-                                   "no target" : (target->obd_stopping ?
-                                   "stopping" : "not set up"));
-                GOTO(out, rc = -ENODEV);
-        }
+       if (!target) {
+               deuuidify(str, NULL, &target_start, &target_len);
+               LCONSOLE_ERROR_MSG(0x137, "UUID '%s' is not available for "
+                                  "connect (no target)\n", str);
+               GOTO(out, rc = -ENODEV);
+       }
+
+       cfs_spin_lock(&target->obd_dev_lock);
+       if (target->obd_stopping || !target->obd_set_up) {
+               cfs_spin_unlock(&target->obd_dev_lock);
+
+               deuuidify(str, NULL, &target_start, &target_len);
+               LCONSOLE_ERROR_MSG(0x137, "%.*s: Not available for connect "
+                                  "from %s (%s)\n", target_len, target_start,
+                                  libcfs_nid2str(req->rq_peer.nid), 
+                                  (target->obd_stopping ?
+                                  "stopping" : "not set up"));
+               GOTO(out, rc = -ENODEV);
+       }

 
         if (target->obd_no_conn) {
 
         if (target->obd_no_conn) {

+               cfs_spin_unlock(&target->obd_dev_lock);
+

                 LCONSOLE_WARN("%s: Temporarily refusing client connection "
                               "from %s\n", target->obd_name,
                               libcfs_nid2str(req->rq_peer.nid));
                 LCONSOLE_WARN("%s: Temporarily refusing client connection "
                               "from %s\n", target->obd_name,
                               libcfs_nid2str(req->rq_peer.nid));


@@ -784,6 +796,8 @@ int target_handle_connect(struct ptlrpc_request *req)
            Really, class_uuid2obd should take the ref. */
         targref = class_incref(target, __FUNCTION__, cfs_current());
 
            Really, class_uuid2obd should take the ref. */
         targref = class_incref(target, __FUNCTION__, cfs_current());
 

+       target->obd_conn_inprogress++;
+       cfs_spin_unlock(&target->obd_dev_lock);

 
         str = req_capsule_client_get(&req->rq_pill, &RMF_CLUUID);
         if (str == NULL) {
 
         str = req_capsule_client_get(&req->rq_pill, &RMF_CLUUID);
         if (str == NULL) {


@@ -1003,19 +1017,12 @@ dont_check_exports:
                         rc = obd_connect(req->rq_svc_thread->t_env,
                                          &export, target, &cluuid, data,
                                          client_nid);
                         rc = obd_connect(req->rq_svc_thread->t_env,
                                          &export, target, &cluuid, data,
                                          client_nid);

-                        if (rc == 0) {
+                        if (rc == 0)

                                 conn.cookie = export->exp_handle.h_cookie;
                                 conn.cookie = export->exp_handle.h_cookie;

-                                /* LU-1092 reconnect put export refcount in the
-                                 * end, connect needs take one here too. */
-                                class_export_get(export);
-                        }

                 }
         } else {
                 rc = obd_reconnect(req->rq_svc_thread->t_env,
                                    export, target, &cluuid, data, client_nid);
                 }
         } else {
                 rc = obd_reconnect(req->rq_svc_thread->t_env,
                                    export, target, &cluuid, data, client_nid);

-                if (rc == 0)
-                        /* prevous done via class_conn2export */
-                        class_export_get(export);

         }
         if (rc)
                 GOTO(out, rc);
         }
         if (rc)
                 GOTO(out, rc);


@@ -1051,7 +1058,8 @@ dont_check_exports:
         if (req->rq_export != NULL)
                 class_export_put(req->rq_export);
 
         if (req->rq_export != NULL)
                 class_export_put(req->rq_export);
 

-        req->rq_export = export;
+       /* request takes one export refcount */
+       req->rq_export = class_export_get(export);

 
         cfs_spin_lock(&export->exp_lock);
         if (export->exp_conn_cnt >= lustre_msg_get_conn_cnt(req->rq_reqmsg)) {
 
         cfs_spin_lock(&export->exp_lock);
         if (export->exp_conn_cnt >= lustre_msg_get_conn_cnt(req->rq_reqmsg)) {


@@ -1198,8 +1206,13 @@ out:
 
                 class_export_put(export);
         }
 
                 class_export_put(export);
         }

-        if (targref)
+        if (targref) {
+               cfs_spin_lock(&target->obd_dev_lock);
+               target->obd_conn_inprogress--;
+               cfs_spin_unlock(&target->obd_dev_lock);
+

                 class_decref(targref, __FUNCTION__, cfs_current());
                 class_decref(targref, __FUNCTION__, cfs_current());

+       }

         if (rc)
                 req->rq_status = rc;
         RETURN(rc);
         if (rc)
                 req->rq_status = rc;
         RETURN(rc);

diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c
index 898ba9c..e0e1fca 100644 (file)
--- a/lustre/obdclass/obd_config.c
+++ b/lustre/obdclass/obd_config.c
@@ -363,6 +363,8 @@ int class_attach(struct lustre_cfg *lcfg)
 
         llog_group_init(&obd->obd_olg, FID_SEQ_LLOG);
 
 
         llog_group_init(&obd->obd_olg, FID_SEQ_LLOG);
 

+       obd->obd_conn_inprogress = 0;
+

         len = strlen(uuid);
         if (len >= sizeof(obd->obd_uuid)) {
                 CERROR("uuid must be < %d bytes long\n",
         len = strlen(uuid);
         if (len >= sizeof(obd->obd_uuid)) {
                 CERROR("uuid must be < %d bytes long\n",


@@ -573,7 +575,16 @@ int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg)
         }
         /* Leave this on forever */
         obd->obd_stopping = 1;
         }
         /* Leave this on forever */
         obd->obd_stopping = 1;

-        cfs_spin_unlock(&obd->obd_dev_lock);
+
+       /* wait for already-arrived-connections to finish. */
+       while (obd->obd_conn_inprogress > 0) {
+               cfs_spin_unlock(&obd->obd_dev_lock);
+
+               cfs_cond_resched();
+
+               cfs_spin_lock(&obd->obd_dev_lock);
+       }
+       cfs_spin_unlock(&obd->obd_dev_lock);

 
         if (lcfg->lcfg_bufcount >= 2 && LUSTRE_CFG_BUFLEN(lcfg, 1) > 0) {
                 for (flag = lustre_cfg_string(lcfg, 1); *flag != 0; flag++)
 
         if (lcfg->lcfg_bufcount >= 2 && LUSTRE_CFG_BUFLEN(lcfg, 1) > 0) {
                 for (flag = lustre_cfg_string(lcfg, 1); *flag != 0; flag++)