From: Liang Zhen Date: Mon, 20 Jan 2014 12:52:51 +0000 (+0800) Subject: LU-4509 ptlrpc: re-enqueue ptlrpcd worker X-Git-Tag: 2.5.55~6 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=d1dcff3084e929f5768dc733cdc104cddf168c06;hp=b8d04564d54fbff0836f3783fd70dcbc8771c008 LU-4509 ptlrpc: re-enqueue ptlrpcd worker osc_extent_wait can be stuck in scenario like this: 1) thread-1 held an active extent 2) thread-2 called flush cache, and marked this extent as "urgent" and "sync_wait" 3) thread-3 wants to write to the same extent, osc_extent_find will get "conflict" because this extent is "sync_wait", so it starts to wait... 4) cl_writeback_work has been scheduled by thread-4 to write some other extents, it has sent RPCs but not returned yet. 5) thread-1 finished his work, and called osc_extent_release()-> osc_io_unplug_async()->ptlrpcd_queue_work(), but found cl_writeback_work is still running, so it's ignored (-EBUSY) 6) thread-3 is stuck because nobody will wake him up. This patch allows ptlrpcd_work to be rescheduled, so it will not miss request anymore Signed-off-by: Liang Zhen Change-Id: I4929d52b2d409c2ce081147bb5ee3dd380a86c43 Reviewed-on: http://review.whamcloud.com/8922 Tested-by: Jenkins Reviewed-by: Jinshan Xiong Reviewed-by: Bobi Jam Tested-by: Maloo Reviewed-by: Oleg Drokin --- diff --git a/lustre/ptlrpc/client.c b/lustre/ptlrpc/client.c index 970e441..d920faa 100644 --- a/lustre/ptlrpc/client.c +++ b/lustre/ptlrpc/client.c @@ -53,6 +53,7 @@ #include "ptlrpc_internal.h" static int ptlrpc_send_new_req(struct ptlrpc_request *req); +static int ptlrpcd_check_work(struct ptlrpc_request *req); /** * Initialize passed in client structure \a cl. @@ -1833,7 +1834,11 @@ int ptlrpc_check_set(const struct lu_env *env, struct ptlrpc_request_set *set) ptlrpc_req_interpret(env, req, req->rq_status); - ptlrpc_rqphase_move(req, RQ_PHASE_COMPLETE); + if (ptlrpcd_check_work(req)) { + atomic_dec(&set->set_remaining); + continue; + } + ptlrpc_rqphase_move(req, RQ_PHASE_COMPLETE); CDEBUG(req->rq_reqmsg != NULL ? D_RPCTRACE : 0, "Completed RPC pname:cluuid:pid:xid:nid:" @@ -3024,22 +3029,50 @@ EXPORT_SYMBOL(ptlrpc_sample_next_xid); * have delay before it really runs by ptlrpcd thread. */ struct ptlrpc_work_async_args { - __u64 magic; - int (*cb)(const struct lu_env *, void *); - void *cbdata; + int (*cb)(const struct lu_env *, void *); + void *cbdata; }; -#define PTLRPC_WORK_MAGIC 0x6655436b676f4f44ULL /* magic code */ +static void ptlrpcd_add_work_req(struct ptlrpc_request *req) +{ + /* re-initialize the req */ + req->rq_timeout = obd_timeout; + req->rq_sent = cfs_time_current_sec(); + req->rq_deadline = req->rq_sent + req->rq_timeout; + req->rq_reply_deadline = req->rq_deadline; + req->rq_phase = RQ_PHASE_INTERPRET; + req->rq_next_phase = RQ_PHASE_COMPLETE; + req->rq_xid = ptlrpc_next_xid(); + req->rq_import_generation = req->rq_import->imp_generation; + + ptlrpcd_add_req(req, PDL_POLICY_ROUND, -1); +} static int work_interpreter(const struct lu_env *env, - struct ptlrpc_request *req, void *data, int rc) + struct ptlrpc_request *req, void *data, int rc) { - struct ptlrpc_work_async_args *arg = data; + struct ptlrpc_work_async_args *arg = data; + + LASSERT(ptlrpcd_check_work(req)); + LASSERT(arg->cb != NULL); + + rc = arg->cb(env, arg->cbdata); + + list_del_init(&req->rq_set_chain); + req->rq_set = NULL; + + if (atomic_dec_return(&req->rq_refcount) > 1) { + atomic_set(&req->rq_refcount, 2); + ptlrpcd_add_work_req(req); + } + return rc; +} - LASSERT(arg->magic == PTLRPC_WORK_MAGIC); - LASSERT(arg->cb != NULL); +static int worker_format; - return arg->cb(env, arg->cbdata); +static int ptlrpcd_check_work(struct ptlrpc_request *req) +{ + return req->rq_pill.rc_fmt == (void *)&worker_format; } /** @@ -3073,6 +3106,7 @@ void *ptlrpcd_alloc_work(struct obd_import *imp, req->rq_receiving_reply = 0; req->rq_must_unlink = 0; req->rq_no_delay = req->rq_no_resend = 1; + req->rq_pill.rc_fmt = (void *)&worker_format; spin_lock_init(&req->rq_lock); CFS_INIT_LIST_HEAD(&req->rq_list); @@ -3082,11 +3116,10 @@ void *ptlrpcd_alloc_work(struct obd_import *imp, CFS_INIT_LIST_HEAD(&req->rq_exp_list); init_waitqueue_head(&req->rq_reply_waitq); init_waitqueue_head(&req->rq_set_waitq); - cfs_atomic_set(&req->rq_refcount, 1); + atomic_set(&req->rq_refcount, 1); CLASSERT (sizeof(*args) <= sizeof(req->rq_async_args)); args = ptlrpc_req_async_args(req); - args->magic = PTLRPC_WORK_MAGIC; args->cb = cb; args->cbdata = cbdata; @@ -3105,7 +3138,7 @@ EXPORT_SYMBOL(ptlrpcd_destroy_work); int ptlrpcd_queue_work(void *handler) { - struct ptlrpc_request *req = handler; + struct ptlrpc_request *req = handler; /* * Check if the req is already being queued. @@ -3115,26 +3148,9 @@ int ptlrpcd_queue_work(void *handler) * for this purpose. This is okay because the caller should use this * req as opaque data. - Jinshan */ - LASSERT(cfs_atomic_read(&req->rq_refcount) > 0); - if (cfs_atomic_read(&req->rq_refcount) > 1) - return -EBUSY; - - if (cfs_atomic_inc_return(&req->rq_refcount) > 2) { /* race */ - cfs_atomic_dec(&req->rq_refcount); - return -EBUSY; - } - - /* re-initialize the req */ - req->rq_timeout = obd_timeout; - req->rq_sent = cfs_time_current_sec(); - req->rq_deadline = req->rq_sent + req->rq_timeout; - req->rq_reply_deadline = req->rq_deadline; - req->rq_phase = RQ_PHASE_INTERPRET; - req->rq_next_phase = RQ_PHASE_COMPLETE; - req->rq_xid = ptlrpc_next_xid(); - req->rq_import_generation = req->rq_import->imp_generation; - - ptlrpcd_add_req(req, PDL_POLICY_ROUND, -1); - return 0; + LASSERT(atomic_read(&req->rq_refcount) > 0); + if (atomic_inc_return(&req->rq_refcount) == 2) + ptlrpcd_add_work_req(req); + return 0; } EXPORT_SYMBOL(ptlrpcd_queue_work);