From 4e2703d8fd7137374cc3f01f721e3291ade20640 Mon Sep 17 00:00:00 2001 From: Lai Siyao Date: Fri, 29 Jun 2012 17:15:41 +0800 Subject: [PATCH] LU-1432 ptlrpc: LBUG in lprocfs_free_client_stats() * serialize connect and target obd cleanup to avoid connect accessing unexisted data structure. * connect export refcounting cleanup. Signed-off-by: Hiroya Nozaki Signed-off-by: Lai Siyao Change-Id: I0a9e8a58ecdc1212565a478f4a758755a1b95f99 Reviewed-on: http://review.whamcloud.com/3244 Tested-by: Hudson Reviewed-by: Bobi Jam Tested-by: Maloo Reviewed-by: Oleg Drokin --- lustre/include/obd.h | 2 ++ lustre/ldlm/ldlm_lib.c | 53 +++++++++++++++++++++++++++----------------- lustre/obdclass/obd_config.c | 13 ++++++++++- 3 files changed, 47 insertions(+), 21 deletions(-) diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 91b5b6e..6c3ec4f 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -1119,6 +1119,8 @@ struct obd_device { * debugging. */ struct lu_ref obd_reference; + + int obd_conn_inprogress; }; #define OBD_LLOG_FL_SENDNOW 0x0001 diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index 4b7e8f2..58b8917 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -700,7 +700,7 @@ target_start_and_reset_recovery_timer(struct obd_device *obd, int target_handle_connect(struct ptlrpc_request *req) { - struct obd_device *target, *targref = NULL; + struct obd_device *target = NULL, *targref = NULL; struct obd_export *export = NULL; struct obd_import *revimp; struct lustre_handle conn; @@ -731,17 +731,29 @@ int target_handle_connect(struct ptlrpc_request *req) if (!target) target = class_name2obd(str); - if (!target || target->obd_stopping || !target->obd_set_up) { - deuuidify(str, NULL, &target_start, &target_len); - LCONSOLE_ERROR_MSG(0x137, "%.*s: Not available for connect " - "from %s (%s)\n", target_len, target_start, - libcfs_nid2str(req->rq_peer.nid), !target ? - "no target" : (target->obd_stopping ? - "stopping" : "not set up")); - GOTO(out, rc = -ENODEV); - } + if (!target) { + deuuidify(str, NULL, &target_start, &target_len); + LCONSOLE_ERROR_MSG(0x137, "UUID '%s' is not available for " + "connect (no target)\n", str); + GOTO(out, rc = -ENODEV); + } + + cfs_spin_lock(&target->obd_dev_lock); + if (target->obd_stopping || !target->obd_set_up) { + cfs_spin_unlock(&target->obd_dev_lock); + + deuuidify(str, NULL, &target_start, &target_len); + LCONSOLE_ERROR_MSG(0x137, "%.*s: Not available for connect " + "from %s (%s)\n", target_len, target_start, + libcfs_nid2str(req->rq_peer.nid), + (target->obd_stopping ? + "stopping" : "not set up")); + GOTO(out, rc = -ENODEV); + } if (target->obd_no_conn) { + cfs_spin_unlock(&target->obd_dev_lock); + LCONSOLE_WARN("%s: Temporarily refusing client connection " "from %s\n", target->obd_name, libcfs_nid2str(req->rq_peer.nid)); @@ -753,6 +765,8 @@ int target_handle_connect(struct ptlrpc_request *req) Really, class_uuid2obd should take the ref. */ targref = class_incref(target, __FUNCTION__, cfs_current()); + target->obd_conn_inprogress++; + cfs_spin_unlock(&target->obd_dev_lock); str = req_capsule_client_get(&req->rq_pill, &RMF_CLUUID); if (str == NULL) { @@ -972,19 +986,12 @@ dont_check_exports: rc = obd_connect(req->rq_svc_thread->t_env, &export, target, &cluuid, data, client_nid); - if (rc == 0) { + if (rc == 0) conn.cookie = export->exp_handle.h_cookie; - /* LU-1092 reconnect put export refcount in the - * end, connect needs take one here too. */ - class_export_get(export); - } } } else { rc = obd_reconnect(req->rq_svc_thread->t_env, export, target, &cluuid, data, client_nid); - if (rc == 0) - /* prevous done via class_conn2export */ - class_export_get(export); } if (rc) GOTO(out, rc); @@ -1016,7 +1023,8 @@ dont_check_exports: if (req->rq_export != NULL) class_export_put(req->rq_export); - req->rq_export = export; + /* request takes one export refcount */ + req->rq_export = class_export_get(export); cfs_spin_lock(&export->exp_lock); if (export->exp_conn_cnt >= lustre_msg_get_conn_cnt(req->rq_reqmsg)) { @@ -1160,8 +1168,13 @@ out: class_export_put(export); } - if (targref) + if (targref) { + cfs_spin_lock(&target->obd_dev_lock); + target->obd_conn_inprogress--; + cfs_spin_unlock(&target->obd_dev_lock); + class_decref(targref, __FUNCTION__, cfs_current()); + } if (rc) req->rq_status = rc; RETURN(rc); diff --git a/lustre/obdclass/obd_config.c b/lustre/obdclass/obd_config.c index 79af076..2b1ad23 100644 --- a/lustre/obdclass/obd_config.c +++ b/lustre/obdclass/obd_config.c @@ -366,6 +366,8 @@ int class_attach(struct lustre_cfg *lcfg) llog_group_init(&obd->obd_olg, FID_SEQ_LLOG); + obd->obd_conn_inprogress = 0; + len = strlen(uuid); if (len >= sizeof(obd->obd_uuid)) { CERROR("uuid must be < %d bytes long\n", @@ -576,7 +578,16 @@ int class_cleanup(struct obd_device *obd, struct lustre_cfg *lcfg) } /* Leave this on forever */ obd->obd_stopping = 1; - cfs_spin_unlock(&obd->obd_dev_lock); + + /* wait for already-arrived-connections to finish. */ + while (obd->obd_conn_inprogress > 0) { + cfs_spin_unlock(&obd->obd_dev_lock); + + cfs_cond_resched(); + + cfs_spin_lock(&obd->obd_dev_lock); + } + cfs_spin_unlock(&obd->obd_dev_lock); if (lcfg->lcfg_bufcount >= 2 && LUSTRE_CFG_BUFLEN(lcfg, 1) > 0) { for (flag = lustre_cfg_string(lcfg, 1); *flag != 0; flag++) -- 1.8.3.1