From 5369adfa13056ee2fcca15fb39e2f8734f28be28 Mon Sep 17 00:00:00 2001 From: yury Date: Mon, 24 Nov 2008 13:34:40 +0000 Subject: [PATCH] b=17631 r=shadow,panda - fixes long bulk unlink done synchronously in ptlrpcd thread which could cause asserion in umount time; --- lustre/ptlrpc/niobuf.c | 86 +++++++++++++++++++++++++++++++------------------- 1 file changed, 54 insertions(+), 32 deletions(-) diff --git a/lustre/ptlrpc/niobuf.c b/lustre/ptlrpc/niobuf.c index 34aa831..276df46d7 100644 --- a/lustre/ptlrpc/niobuf.c +++ b/lustre/ptlrpc/niobuf.c @@ -96,7 +96,7 @@ static int ptl_send_buf (lnet_handle_md_t *mdh, void *base, int len, RETURN (0); } -int ptlrpc_start_bulk_transfer (struct ptlrpc_bulk_desc *desc) +int ptlrpc_start_bulk_transfer(struct ptlrpc_bulk_desc *desc) { struct ptlrpc_connection *conn = desc->bd_export->exp_connection; int rc; @@ -164,16 +164,17 @@ int ptlrpc_start_bulk_transfer (struct ptlrpc_bulk_desc *desc) RETURN(0); } -void ptlrpc_abort_bulk (struct ptlrpc_bulk_desc *desc) +/* Server side bulk abort. Idempotent. Not thread-safe (i.e. only + * serialises with completion callback) */ +void ptlrpc_abort_bulk(struct ptlrpc_request *req) { - /* Server side bulk abort. Idempotent. Not thread-safe (i.e. only - * serialises with completion callback) */ - struct l_wait_info lwi; - int rc; + struct ptlrpc_bulk_desc *desc = req->rq_bulk; + struct l_wait_info lwi; + int rc; - LASSERT (!in_interrupt ()); /* might sleep */ + LASSERT(!in_interrupt()); /* might sleep */ - if (!ptlrpc_bulk_active(desc)) /* completed or */ + if (!ptlrpc_client_bulk_active(req)) /* completed or */ return; /* never started */ /* Do not send any meaningful data over the wire for evicted clients */ @@ -185,14 +186,15 @@ void ptlrpc_abort_bulk (struct ptlrpc_bulk_desc *desc) * but we must still l_wait_event() in this case, to give liblustre * a chance to run server_bulk_callback()*/ - LNetMDUnlink (desc->bd_md_h); + LNetMDUnlink(desc->bd_md_h); for (;;) { /* Network access will complete in finite time but the HUGE * timeout lets us CWARN for visibility of sluggish NALs */ - lwi = LWI_TIMEOUT (cfs_time_seconds(300), NULL, NULL); + lwi = LWI_TIMEOUT_INTERVAL(cfs_time_seconds(LONG_UNLINK), + cfs_time_seconds(1), NULL, NULL); rc = l_wait_event(desc->bd_waitq, - !ptlrpc_bulk_active(desc), &lwi); + !ptlrpc_client_bulk_active(req), &lwi); if (rc == 0) return; @@ -201,7 +203,7 @@ void ptlrpc_abort_bulk (struct ptlrpc_bulk_desc *desc) } } -int ptlrpc_register_bulk (struct ptlrpc_request *req) +int ptlrpc_register_bulk(struct ptlrpc_request *req) { struct ptlrpc_bulk_desc *desc = req->rq_bulk; lnet_process_id_t peer; @@ -275,29 +277,45 @@ int ptlrpc_register_bulk (struct ptlrpc_request *req) RETURN(0); } -void ptlrpc_unregister_bulk (struct ptlrpc_request *req) +/* Disconnect a bulk desc from the network. Idempotent. Not + * thread-safe (i.e. only interlocks with completion callback). */ +int ptlrpc_unregister_bulk(struct ptlrpc_request *req, int async) { - /* Disconnect a bulk desc from the network. Idempotent. Not - * thread-safe (i.e. only interlocks with completion callback). */ struct ptlrpc_bulk_desc *desc = req->rq_bulk; cfs_waitq_t *wq; struct l_wait_info lwi; int rc; + ENTRY; + + LASSERT(!in_interrupt()); /* might sleep */ - LASSERT (!in_interrupt ()); /* might sleep */ + /* Let's setup deadline for reply unlink. */ + if (OBD_FAIL_CHECK(OBD_FAIL_PTLRPC_LONG_BULK_UNLINK) && + async && req->rq_bulk_deadline == 0) + req->rq_bulk_deadline = cfs_time_current_sec() + LONG_UNLINK; - if (!ptlrpc_bulk_active(desc)) /* completed or */ - return; /* never registered */ + if (!ptlrpc_client_bulk_active(req)) /* completed or */ + RETURN(1); /* never registered */ - LASSERT (desc->bd_req == req); /* bd_req NULL until registered */ + LASSERT(desc->bd_req == req); /* bd_req NULL until registered */ /* the unlink ensures the callback happens ASAP and is the last * one. If it fails, it must be because completion just happened, * but we must still l_wait_event() in this case to give liblustre * a chance to run client_bulk_callback() */ - LNetMDUnlink (desc->bd_md_h); - + LNetMDUnlink(desc->bd_md_h); + + if (!ptlrpc_client_bulk_active(req)) /* completed or */ + RETURN(1); /* never registered */ + + /* Move to "Unregistering" phase as bulk was not unlinked yet. */ + ptlrpc_rqphase_move(req, RQ_PHASE_UNREGISTERING); + + /* Do not wait for unlink to finish. */ + if (async) + RETURN(0); + if (req->rq_set != NULL) wq = &req->rq_set->set_waitq; else @@ -306,18 +324,22 @@ void ptlrpc_unregister_bulk (struct ptlrpc_request *req) for (;;) { /* Network access will complete in finite time but the HUGE * timeout lets us CWARN for visibility of sluggish NALs */ - lwi = LWI_TIMEOUT (cfs_time_seconds(300), NULL, NULL); - rc = l_wait_event(*wq, !ptlrpc_bulk_active(desc), &lwi); - if (rc == 0) - return; + lwi = LWI_TIMEOUT_INTERVAL(cfs_time_seconds(LONG_UNLINK), + cfs_time_seconds(1), NULL, NULL); + rc = l_wait_event(*wq, !ptlrpc_client_bulk_active(req), &lwi); + if (rc == 0) { + ptlrpc_rqphase_move(req, req->rq_next_phase); + RETURN(1); + } - LASSERT (rc == -ETIMEDOUT); - DEBUG_REQ(D_WARNING,req,"Unexpectedly long timeout: desc %p", + LASSERT(rc == -ETIMEDOUT); + DEBUG_REQ(D_WARNING, req, "Unexpectedly long timeout: desc %p", desc); } + RETURN(0); } -int ptlrpc_send_reply (struct ptlrpc_request *req, int flags) +int ptlrpc_send_reply(struct ptlrpc_request *req, int flags) { struct ptlrpc_service *svc = req->rq_rqbd->rqbd_service; struct ptlrpc_reply_state *rs = req->rq_reply_state; @@ -619,13 +641,13 @@ int ptl_send_rpc(struct ptlrpc_request *request, int noreply) request->rq_repmsg = NULL; //remove cleanup_bulk: - if (request->rq_bulk != NULL) - ptlrpc_unregister_bulk(request); - + /* We do sync unlink here as there was no real transfer here so + * the chance to have long unlink to sluggish net is smaller here. */ + ptlrpc_unregister_bulk(request, 0); return rc; } -int ptlrpc_register_rqbd (struct ptlrpc_request_buffer_desc *rqbd) +int ptlrpc_register_rqbd(struct ptlrpc_request_buffer_desc *rqbd) { struct ptlrpc_service *service = rqbd->rqbd_service; static lnet_process_id_t match_id = {LNET_NID_ANY, LNET_PID_ANY}; -- 1.8.3.1