From 49b60fed44263fc791bf8fc3dc81f6ea359a58bf Mon Sep 17 00:00:00 2001 From: wangdi Date: Fri, 4 Jul 2008 05:17:27 +0000 Subject: [PATCH] Branch: b1_6 sync import destroy with llog thread/lprocfs access b=14629 i=robert,jay --- lustre/ChangeLog | 6 +++++ lustre/include/lprocfs_status.h | 6 ++--- lustre/include/obd.h | 2 +- lustre/ldlm/ldlm_lib.c | 10 +++---- lustre/mdc/mdc_request.c | 29 +++++++++++++++------ lustre/mgc/libmgc.c | 2 +- lustre/mgc/mgc_request.c | 6 ++--- lustre/obdclass/llog_obd.c | 4 +++ lustre/osc/osc_request.c | 21 ++++++++++++--- lustre/ptlrpc/llog_client.c | 58 ++++++++++++++++++++++++++++++----------- lustre/ptlrpc/llog_net.c | 19 ++++++++++++-- lustre/ptlrpc/recov_thread.c | 1 + 12 files changed, 122 insertions(+), 42 deletions(-) diff --git a/lustre/ChangeLog b/lustre/ChangeLog index d95af11..47b10c1 100644 --- a/lustre/ChangeLog +++ b/lustre/ChangeLog @@ -251,6 +251,12 @@ Details : The direct IO path doesn't call check_rpcs to submit a new RPC once one is completed. As a result, some RPCs are stuck in the queue and are never sent. +Severity : normal +Bugzilla : 15684 +Description: Procfs and llog threads access destoryed import sometimes. +Details : Sync the import destoryed process with procfs and llog threads by + the import refcount and semaphore. + ------------------------------------------------------------------------------- diff --git a/lustre/include/lprocfs_status.h b/lustre/include/lprocfs_status.h index 842576f..688fbdc 100644 --- a/lustre/include/lprocfs_status.h +++ b/lustre/include/lprocfs_status.h @@ -493,14 +493,14 @@ extern struct rw_semaphore _lprocfs_lock; * the import in a client obd_device for a lprocfs entry */ #define LPROCFS_CLIMP_CHECK(obd) do { \ typecheck(struct obd_device *, obd); \ - mutex_down(&(obd)->u.cli.cl_sem); \ + down_read(&(obd)->u.cli.cl_sem); \ if ((obd)->u.cli.cl_import == NULL) { \ - mutex_up(&(obd)->u.cli.cl_sem); \ + up_read(&(obd)->u.cli.cl_sem); \ return -ENODEV; \ } \ } while(0) #define LPROCFS_CLIMP_EXIT(obd) \ - mutex_up(&(obd)->u.cli.cl_sem); + up_read(&(obd)->u.cli.cl_sem); /* write the name##_seq_show function, call LPROC_SEQ_FOPS_RO for read-only diff --git a/lustre/include/obd.h b/lustre/include/obd.h index 5f8cf4d..7267fba 100644 --- a/lustre/include/obd.h +++ b/lustre/include/obd.h @@ -337,7 +337,7 @@ struct mdc_rpc_lock; struct obd_import; struct lustre_cache; struct client_obd { - struct semaphore cl_sem; + struct rw_semaphore cl_sem; struct obd_uuid cl_target_uuid; struct obd_import *cl_import; /* ptlrpc connection state */ int cl_conn_count; diff --git a/lustre/ldlm/ldlm_lib.c b/lustre/ldlm/ldlm_lib.c index 648cd0e..c5293b3 100644 --- a/lustre/ldlm/ldlm_lib.c +++ b/lustre/ldlm/ldlm_lib.c @@ -239,7 +239,7 @@ int client_obd_setup(struct obd_device *obddev, obd_count len, void *buf) RETURN(-EINVAL); } - sema_init(&cli->cl_sem, 1); + init_rwsem(&cli->cl_sem); sema_init(&cli->cl_mgc_sem, 1); cli->cl_conn_count = 0; memcpy(server_uuid.uuid, lustre_cfg_buf(lcfg, 2), @@ -371,7 +371,7 @@ int client_connect_import(struct lustre_handle *dlm_handle, int rc; ENTRY; - mutex_down(&cli->cl_sem); + down_write(&cli->cl_sem); rc = class_connect(dlm_handle, obd, cluuid); if (rc) GOTO(out_sem, rc); @@ -428,7 +428,7 @@ out_disco: class_export_put(exp); } out_sem: - mutex_up(&cli->cl_sem); + up_write(&cli->cl_sem); if (to_be_freed) ldlm_namespace_free_post(to_be_freed); return rc; @@ -452,7 +452,7 @@ int client_disconnect_export(struct obd_export *exp) cli = &obd->u.cli; imp = cli->cl_import; - mutex_down(&cli->cl_sem); + down_write(&cli->cl_sem); if (!cli->cl_conn_count) { CERROR("disconnecting disconnected device (%s)\n", obd->obd_name); @@ -503,7 +503,7 @@ int client_disconnect_export(struct obd_export *exp) if (!rc && err) rc = err; out_sem: - mutex_up(&cli->cl_sem); + up_write(&cli->cl_sem); if (to_be_freed) ldlm_namespace_free_post(to_be_freed); RETURN(rc); diff --git a/lustre/mdc/mdc_request.c b/lustre/mdc/mdc_request.c index 0073231..51cc8bc 100644 --- a/lustre/mdc/mdc_request.c +++ b/lustre/mdc/mdc_request.c @@ -982,19 +982,30 @@ static int mdc_statfs(struct obd_device *obd, struct obd_statfs *osfs, { struct ptlrpc_request *req; struct obd_statfs *msfs; + struct obd_import *imp = NULL; int rc, size[2] = { sizeof(struct ptlrpc_body), sizeof(*msfs) }; ENTRY; + /*Since the request might also come from lprocfs, so we need + *sync this with client_disconnect_export Bug15684*/ + down_read(&obd->u.cli.cl_sem); + if (obd->u.cli.cl_import) + imp = class_import_get(obd->u.cli.cl_import); + up_read(&obd->u.cli.cl_sem); + if (!imp) + RETURN(-ENODEV); + + /* We could possibly pass max_age in the request (as an absolute * timestamp or a "seconds.usec ago") so the target can avoid doing * extra calls into the filesystem if that isn't necessary (e.g. * during mount that would help a bit). Having relative timestamps * is not so great if request processing is slow, while absolute * timestamps are not ideal because they need time synchronization. */ - req = ptlrpc_prep_req(obd->u.cli.cl_import, LUSTRE_MDS_VERSION, - MDS_STATFS, 1, NULL, NULL); + req = ptlrpc_prep_req(imp, LUSTRE_MDS_VERSION, MDS_STATFS, 1, NULL, + NULL); if (!req) - RETURN(-ENOMEM); + GOTO(output, rc = -ENOMEM); ptlrpc_req_set_repsize(req, 2, size); @@ -1020,7 +1031,8 @@ static int mdc_statfs(struct obd_device *obd, struct obd_statfs *osfs, EXIT; out: ptlrpc_req_finished(req); - +output: + class_import_put(imp); return rc; } @@ -1281,11 +1293,12 @@ static int mdc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) class_destroy_import(imp); obd->u.cli.cl_import = NULL; } - break; - case OBD_CLEANUP_SELF_EXP: rc = obd_llog_finish(obd, 0); if (rc != 0) CERROR("failed to cleanup llogging subsystems\n"); + break; + case OBD_CLEANUP_SELF_EXP: + break; case OBD_CLEANUP_OBD: break; } @@ -1320,7 +1333,7 @@ static int mdc_llog_init(struct obd_device *obd, struct obd_device *tgt, &llog_client_ops); if (rc == 0) { ctxt = llog_get_context(obd, LLOG_CONFIG_REPL_CTXT); - ctxt->loc_imp = obd->u.cli.cl_import; + llog_initiator_connect(ctxt); llog_ctxt_put(ctxt); } @@ -1328,7 +1341,7 @@ static int mdc_llog_init(struct obd_device *obd, struct obd_device *tgt, &llog_client_ops); if (rc == 0) { ctxt = llog_get_context(obd, LLOG_LOVEA_REPL_CTXT); - ctxt->loc_imp = obd->u.cli.cl_import; + llog_initiator_connect(ctxt); llog_ctxt_put(ctxt); } diff --git a/lustre/mgc/libmgc.c b/lustre/mgc/libmgc.c index b72e8bb..c8fd0d5 100644 --- a/lustre/mgc/libmgc.c +++ b/lustre/mgc/libmgc.c @@ -111,7 +111,7 @@ static int mgc_llog_init(struct obd_device *obd, struct obd_device *tgt, &llog_client_ops); if (rc == 0) { ctxt = llog_get_context(obd, LLOG_CONFIG_REPL_CTXT); - ctxt->loc_imp = obd->u.cli.cl_import; + llog_initiator_connect(ctxt); llog_ctxt_put(ctxt); } diff --git a/lustre/mgc/mgc_request.c b/lustre/mgc/mgc_request.c index 9b06f72..660229d 100644 --- a/lustre/mgc/mgc_request.c +++ b/lustre/mgc/mgc_request.c @@ -469,12 +469,12 @@ static int mgc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) spin_unlock(&config_list_lock); cfs_waitq_signal(&rq_waitq); } - break; - case OBD_CLEANUP_SELF_EXP: rc = obd_llog_finish(obd, 0); if (rc != 0) CERROR("failed to cleanup llogging subsystems\n"); break; + case OBD_CLEANUP_SELF_EXP: + break; case OBD_CLEANUP_OBD: break; } @@ -921,7 +921,7 @@ static int mgc_llog_init(struct obd_device *obd, struct obd_device *tgt, &llog_client_ops); if (rc == 0) { ctxt = llog_get_context(obd, LLOG_CONFIG_REPL_CTXT); - ctxt->loc_imp = obd->u.cli.cl_import; + llog_initiator_connect(ctxt); llog_ctxt_put(ctxt); } diff --git a/lustre/obdclass/llog_obd.c b/lustre/obdclass/llog_obd.c index c9e9a95..a49c621 100644 --- a/lustre/obdclass/llog_obd.c +++ b/lustre/obdclass/llog_obd.c @@ -56,6 +56,10 @@ static void llog_ctxt_destroy(struct llog_ctxt *ctxt) { if (ctxt->loc_exp) class_export_put(ctxt->loc_exp); + if (ctxt->loc_imp) { + class_import_put(ctxt->loc_imp); + ctxt->loc_imp = NULL; + } OBD_FREE(ctxt, sizeof(*ctxt)); return; } diff --git a/lustre/osc/osc_request.c b/lustre/osc/osc_request.c index 9c5a2db..a4f7394 100644 --- a/lustre/osc/osc_request.c +++ b/lustre/osc/osc_request.c @@ -3236,17 +3236,29 @@ static int osc_statfs(struct obd_device *obd, struct obd_statfs *osfs, { struct obd_statfs *msfs; struct ptlrpc_request *req; + struct obd_import *imp = NULL; int rc, size[2] = { sizeof(struct ptlrpc_body), sizeof(*osfs) }; ENTRY; + /*Since the request might also come from lprocfs, so we need + *sync this with client_disconnect_export Bug15684*/ + down_read(&obd->u.cli.cl_sem); + if (obd->u.cli.cl_import) + imp = class_import_get(obd->u.cli.cl_import); + up_read(&obd->u.cli.cl_sem); + if (!imp) + RETURN(-ENODEV); + /* We could possibly pass max_age in the request (as an absolute * timestamp or a "seconds.usec ago") so the target can avoid doing * extra calls into the filesystem if that isn't necessary (e.g. * during mount that would help a bit). Having relative timestamps * is not so great if request processing is slow, while absolute * timestamps are not ideal because they need time synchronization. */ - req = ptlrpc_prep_req(obd->u.cli.cl_import, LUSTRE_OST_VERSION, + req = ptlrpc_prep_req(imp, LUSTRE_OST_VERSION, OST_STATFS, 1, NULL, NULL); + + class_import_put(imp); if (!req) RETURN(-ENOMEM); @@ -3742,6 +3754,7 @@ static int osc_import_event(struct obd_device *obd, oscc->oscc_flags &= ~OSCC_FLAG_NOSPC; spin_unlock(&oscc->oscc_lock); } + CDEBUG(D_INFO, "notify server \n"); rc = obd_notify_observer(obd, obd, OBD_NOTIFY_ACTIVE, NULL); break; } @@ -3836,13 +3849,13 @@ static int osc_precleanup(struct obd_device *obd, enum obd_cleanup_stage stage) class_destroy_import(imp); obd->u.cli.cl_import = NULL; } - break; - } - case OBD_CLEANUP_SELF_EXP: rc = obd_llog_finish(obd, 0); if (rc != 0) CERROR("failed to cleanup llogging subsystems\n"); break; + } + case OBD_CLEANUP_SELF_EXP: + break; case OBD_CLEANUP_OBD: break; } diff --git a/lustre/ptlrpc/llog_client.c b/lustre/ptlrpc/llog_client.c index 27578a4..ff5fddd 100644 --- a/lustre/ptlrpc/llog_client.c +++ b/lustre/ptlrpc/llog_client.c @@ -43,6 +43,31 @@ #include #include +#define LLOG_CLIENT_ENTRY(ctxt, imp) do { \ + mutex_down(&ctxt->loc_sem); \ + if (ctxt->loc_imp) { \ + imp = class_import_get(ctxt->loc_imp); \ + } else { \ + CERROR("ctxt->loc_imp == NULL for context idx %d." \ + "Unable to complete MDS/OSS recovery," \ + "but I'll try again next time. Not fatal.\n", \ + ctxt->loc_idx); \ + imp = NULL; \ + mutex_up(&ctxt->loc_sem); \ + return (-EINVAL); \ + } \ + mutex_up(&ctxt->loc_sem); \ +} while(0) + +#define LLOG_CLIENT_EXIT(ctxt, imp) do { \ + mutex_down(&ctxt->loc_sem); \ + if (ctxt->loc_imp != imp) \ + CWARN("loc_imp has changed from %p to %p", \ + ctxt->loc_imp, imp); \ + class_import_put(imp); \ + mutex_up(&ctxt->loc_sem); \ +} while(0) + /* This is a callback from the llog_* functions. * Assumes caller has already pushed us into the kernel context. */ static int llog_client_create(struct llog_ctxt *ctxt, struct llog_handle **res, @@ -59,18 +84,11 @@ static int llog_client_create(struct llog_ctxt *ctxt, struct llog_handle **res, int rc; ENTRY; - if (ctxt->loc_imp == NULL) { - /* This used to be an assert; bug 6200 */ - CERROR("ctxt->loc_imp == NULL for context idx %d. Unable to " - "complete MDS/OSS recovery, but I'll try again next " - "time. Not fatal.\n", ctxt->loc_idx); - RETURN(-EINVAL); - } - imp = ctxt->loc_imp; + LLOG_CLIENT_ENTRY(ctxt, imp); handle = llog_alloc_handle(); if (handle == NULL) - RETURN(-ENOMEM); + GOTO(out, rc = -ENOMEM); *res = handle; memset(&req_body, 0, sizeof(req_body)); @@ -107,6 +125,7 @@ static int llog_client_create(struct llog_ctxt *ctxt, struct llog_handle **res, out: if (req) ptlrpc_req_finished(req); + LLOG_CLIENT_EXIT(ctxt, imp); RETURN(rc); err_free: @@ -116,17 +135,18 @@ err_free: static int llog_client_destroy(struct llog_handle *loghandle) { - struct obd_import *imp = loghandle->lgh_ctxt->loc_imp; + struct obd_import *imp; struct ptlrpc_request *req = NULL; struct llogd_body *body; int size[] = { sizeof(struct ptlrpc_body), sizeof(*body) }; int rc; ENTRY; + LLOG_CLIENT_ENTRY(loghandle->lgh_ctxt, imp); req = ptlrpc_prep_req(imp, LUSTRE_LOG_VERSION, LLOG_ORIGIN_HANDLE_DESTROY, 2, size, NULL); if (!req) - RETURN(-ENOMEM); + GOTO(out, rc = -ENOMEM); body = lustre_msg_buf(req->rq_reqmsg, REQ_REC_OFF, sizeof(*body)); body->lgd_logid = loghandle->lgh_id; @@ -136,6 +156,8 @@ static int llog_client_destroy(struct llog_handle *loghandle) rc = ptlrpc_queue_wait(req); ptlrpc_req_finished(req); +out: + LLOG_CLIENT_EXIT(loghandle->lgh_ctxt, imp); RETURN(rc); } @@ -144,7 +166,7 @@ static int llog_client_next_block(struct llog_handle *loghandle, int *cur_idx, int next_idx, __u64 *cur_offset, void *buf, int len) { - struct obd_import *imp = loghandle->lgh_ctxt->loc_imp; + struct obd_import *imp; struct ptlrpc_request *req = NULL; struct llogd_body *body; void * ptr; @@ -152,10 +174,11 @@ static int llog_client_next_block(struct llog_handle *loghandle, int rc; ENTRY; + LLOG_CLIENT_ENTRY(loghandle->lgh_ctxt, imp); req = ptlrpc_prep_req(imp, LUSTRE_LOG_VERSION, LLOG_ORIGIN_HANDLE_NEXT_BLOCK, 2, size, NULL); if (!req) - GOTO(out, rc = -ENOMEM); + GOTO(out, rc =-ENOMEM); body = lustre_msg_buf(req->rq_reqmsg, REQ_REC_OFF, sizeof(*body)); body->lgd_logid = loghandle->lgh_id; @@ -194,13 +217,14 @@ static int llog_client_next_block(struct llog_handle *loghandle, out: if (req) ptlrpc_req_finished(req); + LLOG_CLIENT_EXIT(loghandle->lgh_ctxt, imp); RETURN(rc); } static int llog_client_prev_block(struct llog_handle *loghandle, int prev_idx, void *buf, int len) { - struct obd_import *imp = loghandle->lgh_ctxt->loc_imp; + struct obd_import *imp; struct ptlrpc_request *req = NULL; struct llogd_body *body; void * ptr; @@ -208,6 +232,7 @@ static int llog_client_prev_block(struct llog_handle *loghandle, int rc; ENTRY; + LLOG_CLIENT_ENTRY(loghandle->lgh_ctxt, imp); req = ptlrpc_prep_req(imp, LUSTRE_LOG_VERSION, LLOG_ORIGIN_HANDLE_PREV_BLOCK, 2, size, NULL); if (!req) @@ -244,12 +269,13 @@ static int llog_client_prev_block(struct llog_handle *loghandle, out: if (req) ptlrpc_req_finished(req); + LLOG_CLIENT_EXIT(loghandle->lgh_ctxt, imp); RETURN(rc); } static int llog_client_read_header(struct llog_handle *handle) { - struct obd_import *imp = handle->lgh_ctxt->loc_imp; + struct obd_import *imp; struct ptlrpc_request *req = NULL; struct llogd_body *body; struct llog_log_hdr *hdr; @@ -259,6 +285,7 @@ static int llog_client_read_header(struct llog_handle *handle) int rc; ENTRY; + LLOG_CLIENT_ENTRY(handle->lgh_ctxt, imp); req = ptlrpc_prep_req(imp, LUSTRE_LOG_VERSION, LLOG_ORIGIN_HANDLE_READ_HEADER, 2, size, NULL); if (!req) @@ -301,6 +328,7 @@ static int llog_client_read_header(struct llog_handle *handle) out: if (req) ptlrpc_req_finished(req); + LLOG_CLIENT_EXIT(handle->lgh_ctxt, imp); RETURN(rc); } diff --git a/lustre/ptlrpc/llog_net.c b/lustre/ptlrpc/llog_net.c index 3779ed5..92b1418 100644 --- a/lustre/ptlrpc/llog_net.c +++ b/lustre/ptlrpc/llog_net.c @@ -147,16 +147,31 @@ int llog_receptor_accept(struct llog_ctxt *ctxt, struct obd_import *imp) { ENTRY; LASSERT(ctxt); - ctxt->loc_imp = imp; + mutex_down(&ctxt->loc_sem); + if (ctxt->loc_imp != imp) { + CWARN("changing the import %p - %p\n", ctxt->loc_imp, imp); + if (ctxt->loc_imp) + class_import_put(ctxt->loc_imp); + ctxt->loc_imp = class_import_get(imp); + } + mutex_up(&ctxt->loc_sem); RETURN(0); } EXPORT_SYMBOL(llog_receptor_accept); int llog_initiator_connect(struct llog_ctxt *ctxt) { + struct obd_import *new_imp; ENTRY; LASSERT(ctxt); - ctxt->loc_imp = ctxt->loc_obd->u.cli.cl_import; + new_imp = ctxt->loc_obd->u.cli.cl_import; + mutex_down(&ctxt->loc_sem); + if (ctxt->loc_imp != new_imp) { + if (ctxt->loc_imp) + class_import_put(ctxt->loc_imp); + ctxt->loc_imp = class_import_get(new_imp); + } + mutex_up(&ctxt->loc_sem); RETURN(0); } EXPORT_SYMBOL(llog_initiator_connect); diff --git a/lustre/ptlrpc/recov_thread.c b/lustre/ptlrpc/recov_thread.c index 35383e7..6bf6145 100644 --- a/lustre/ptlrpc/recov_thread.c +++ b/lustre/ptlrpc/recov_thread.c @@ -168,6 +168,7 @@ static void ctxt_llcd_put(struct llog_ctxt *ctxt) llcd_put(ctxt->loc_llcd); ctxt->loc_llcd = NULL; } + class_import_put(ctxt->loc_imp); ctxt->loc_imp = NULL; mutex_up(&ctxt->loc_sem); } -- 1.8.3.1