From: Liang Zhen Date: Sat, 26 May 2012 06:40:05 +0000 (+0800) Subject: LU-56 ptlrpc: cleanup of ptlrpc_unregister_service X-Git-Tag: 2.2.58~7 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=582c110231cf06bcd7e5e0b3bdf4f2058e18ebe4 LU-56 ptlrpc: cleanup of ptlrpc_unregister_service This patch is only a cleanup of ptlrpc_unregister_service(), it's an intermediate patch for partitioned ptlrpc service. Signed-off-by: Liang Zhen Change-Id: I334fa3813f77711defd598bbb7f688d3ea6026be Reviewed-on: http://review.whamcloud.com/2917 Tested-by: Hudson Tested-by: Maloo Reviewed-by: wangdi Reviewed-by: Andreas Dilger --- diff --git a/lustre/ptlrpc/service.c b/lustre/ptlrpc/service.c index e6fe4c7..c95f76e 100644 --- a/lustre/ptlrpc/service.c +++ b/lustre/ptlrpc/service.c @@ -2737,151 +2737,211 @@ static void ptlrpc_wait_replies(struct ptlrpc_service_part *svcpt) } } -int ptlrpc_unregister_service(struct ptlrpc_service *service) +static void +ptlrpc_service_del_atimer(struct ptlrpc_service *svc) { - struct l_wait_info lwi; - struct ptlrpc_service_part *svcpt; - struct ptlrpc_reply_state *rs; - struct ptlrpc_reply_state *t; - struct ptlrpc_at_array *array; - cfs_list_t *tmp; - int rc; - ENTRY; - - service->srv_is_stopping = 1; - svcpt = service->srv_part; + struct ptlrpc_service_part *svcpt; - if (svcpt == NULL || /* no instance of ptlrpc_service_part */ - svcpt->scp_service == NULL) /* it's not fully initailzed */ - GOTO(out, rc = 0); - - cfs_timer_disarm(&svcpt->scp_at_timer); - - ptlrpc_stop_all_threads(service); + /* early disarm AT timer... */ + do { /* iterrate over multiple partitions in the future */ + svcpt = svc->srv_part; + if (svcpt == NULL || svcpt->scp_service == NULL) + break; - cfs_spin_lock (&ptlrpc_all_services_lock); - cfs_list_del_init (&service->srv_list); - cfs_spin_unlock (&ptlrpc_all_services_lock); + cfs_timer_disarm(&svcpt->scp_at_timer); + } while (0); +} - ptlrpc_lprocfs_unregister_service(service); +static void +ptlrpc_service_unlink_rqbd(struct ptlrpc_service *svc) +{ + struct ptlrpc_service_part *svcpt; + struct ptlrpc_request_buffer_desc *rqbd; + struct l_wait_info lwi; + int rc; /* All history will be culled when the next request buffer is - * freed */ - service->srv_max_history_rqbds = 0; + * freed in ptlrpc_service_purge_all() */ + svc->srv_max_history_rqbds = 0; - CDEBUG(D_NET, "%s: tearing down\n", service->srv_name); + rc = LNetClearLazyPortal(svc->srv_req_portal); + LASSERT(rc == 0); - rc = LNetClearLazyPortal(service->srv_req_portal); - LASSERT (rc == 0); + do { /* iterrate over multiple partitions in the future */ + svcpt = svc->srv_part; + if (svcpt == NULL || svcpt->scp_service == NULL) + break; - /* Unlink all the request buffers. This forces a 'final' event with - * its 'unlink' flag set for each posted rqbd */ - cfs_list_for_each(tmp, &svcpt->scp_rqbd_posted) { - struct ptlrpc_request_buffer_desc *rqbd = - cfs_list_entry(tmp, struct ptlrpc_request_buffer_desc, - rqbd_list); + /* Unlink all the request buffers. This forces a 'final' + * event with its 'unlink' flag set for each posted rqbd */ + cfs_list_for_each_entry(rqbd, &svcpt->scp_rqbd_posted, + rqbd_list) { + rc = LNetMDUnlink(rqbd->rqbd_md_h); + LASSERT(rc == 0 || rc == -ENOENT); + } + } while (0); - rc = LNetMDUnlink(rqbd->rqbd_md_h); - LASSERT (rc == 0 || rc == -ENOENT); - } + do { /* iterrate over multiple partitions in the future */ + svcpt = svc->srv_part; + if (svcpt == NULL || svcpt->scp_service == NULL) + break; - /* Wait for the network to release any buffers it's currently - * filling */ - for (;;) { + /* Wait for the network to release any buffers + * it's currently filling */ cfs_spin_lock(&svcpt->scp_lock); - rc = svcpt->scp_nrqbds_posted; + while (svcpt->scp_nrqbds_posted != 0) { + cfs_spin_unlock(&svcpt->scp_lock); + /* Network access will complete in finite time but + * the HUGE timeout lets us CWARN for visibility + * of sluggish NALs */ + lwi = LWI_TIMEOUT_INTERVAL( + cfs_time_seconds(LONG_UNLINK), + cfs_time_seconds(1), NULL, NULL); + rc = l_wait_event(svcpt->scp_waitq, + svcpt->scp_nrqbds_posted == 0, &lwi); + if (rc == -ETIMEDOUT) { + CWARN("Service %s waiting for " + "request buffers\n", + svcpt->scp_service->srv_name); + } + cfs_spin_lock(&svcpt->scp_lock); + } cfs_spin_unlock(&svcpt->scp_lock); + } while (0); +} - if (rc == 0) - break; +static void +ptlrpc_service_purge_all(struct ptlrpc_service *svc) +{ + struct ptlrpc_service_part *svcpt; + struct ptlrpc_request_buffer_desc *rqbd; + struct ptlrpc_request *req; + struct ptlrpc_reply_state *rs; - /* Network access will complete in finite time but the HUGE - * timeout lets us CWARN for visibility of sluggish NALs */ - lwi = LWI_TIMEOUT_INTERVAL(cfs_time_seconds(LONG_UNLINK), - cfs_time_seconds(1), NULL, NULL); - rc = l_wait_event(svcpt->scp_waitq, - svcpt->scp_nrqbds_posted == 0, &lwi); - if (rc == -ETIMEDOUT) - CWARN("Service %s waiting for request buffers\n", - service->srv_name); - } + do { /* iterrate over multiple partitions in the future */ + /* schedule all outstanding replies to terminate them */ + svcpt = svc->srv_part; + if (svcpt == NULL || svcpt->scp_service == NULL) + break; - /* schedule all outstanding replies to terminate them */ - cfs_spin_lock(&svcpt->scp_rep_lock); - while (!cfs_list_empty(&svcpt->scp_rep_active)) { - struct ptlrpc_reply_state *rs = - cfs_list_entry(svcpt->scp_rep_active.next, - struct ptlrpc_reply_state, rs_list); - cfs_spin_lock(&rs->rs_lock); - ptlrpc_schedule_difficult_reply(rs); - cfs_spin_unlock(&rs->rs_lock); - } - cfs_spin_unlock(&svcpt->scp_rep_lock); + cfs_spin_lock(&svcpt->scp_rep_lock); + while (!cfs_list_empty(&svcpt->scp_rep_active)) { + rs = cfs_list_entry(svcpt->scp_rep_active.next, + struct ptlrpc_reply_state, rs_list); + cfs_spin_lock(&rs->rs_lock); + ptlrpc_schedule_difficult_reply(rs); + cfs_spin_unlock(&rs->rs_lock); + } + cfs_spin_unlock(&svcpt->scp_rep_lock); + + /* purge the request queue. NB No new replies (rqbds + * all unlinked) and no service threads, so I'm the only + * thread noodling the request queue now */ + while (!cfs_list_empty(&svcpt->scp_req_incoming)) { + req = cfs_list_entry(svcpt->scp_req_incoming.next, + struct ptlrpc_request, rq_list); + + cfs_list_del(&req->rq_list); + svcpt->scp_nreqs_incoming--; + svcpt->scp_nreqs_active++; + ptlrpc_server_finish_request(svcpt, req); + } - /* purge the request queue. NB No new replies (rqbds all unlinked) - * and no service threads, so I'm the only thread noodling the - * request queue now */ - while (!cfs_list_empty(&svcpt->scp_req_incoming)) { - struct ptlrpc_request *req = - cfs_list_entry(svcpt->scp_req_incoming.next, - struct ptlrpc_request, - rq_list); + while (ptlrpc_server_request_pending(svcpt, 1)) { + req = ptlrpc_server_request_get(svcpt, 1); + cfs_list_del(&req->rq_list); + svcpt->scp_nreqs_active++; + ptlrpc_hpreq_fini(req); + ptlrpc_server_finish_request(svcpt, req); + } - cfs_list_del(&req->rq_list); - svcpt->scp_nreqs_incoming--; - svcpt->scp_nreqs_active++; - ptlrpc_server_finish_request(svcpt, req); - } - while (ptlrpc_server_request_pending(svcpt, 1)) { - struct ptlrpc_request *req; + LASSERT(cfs_list_empty(&svcpt->scp_rqbd_posted)); + LASSERT(svcpt->scp_nreqs_incoming == 0); + LASSERT(svcpt->scp_nreqs_active == 0); + /* history should have been culled by + * ptlrpc_server_finish_request */ + LASSERT(svcpt->scp_hist_nrqbds == 0); - req = ptlrpc_server_request_get(svcpt, 1); - cfs_list_del(&req->rq_list); - svcpt->scp_nreqs_active++; - ptlrpc_server_finish_request(svcpt, req); - } - LASSERT(svcpt->scp_nreqs_incoming == 0); - LASSERT(svcpt->scp_nreqs_active == 0); - LASSERT(svcpt->scp_hist_nrqbds == 0); - LASSERT(cfs_list_empty(&svcpt->scp_rqbd_posted)); - - /* Now free all the request buffers since nothing references them - * any more... */ - while (!cfs_list_empty(&svcpt->scp_rqbd_idle)) { - struct ptlrpc_request_buffer_desc *rqbd = - cfs_list_entry(svcpt->scp_rqbd_idle.next, - struct ptlrpc_request_buffer_desc, - rqbd_list); - - ptlrpc_free_rqbd(rqbd); - } + /* Now free all the request buffers since nothing + * references them any more... */ - ptlrpc_wait_replies(svcpt); + while (!cfs_list_empty(&svcpt->scp_rqbd_idle)) { + rqbd = cfs_list_entry(svcpt->scp_rqbd_idle.next, + struct ptlrpc_request_buffer_desc, + rqbd_list); + ptlrpc_free_rqbd(rqbd); + } + ptlrpc_wait_replies(svcpt); + + while (!cfs_list_empty(&svcpt->scp_rep_idle)) { + rs = cfs_list_entry(svcpt->scp_rep_idle.next, + struct ptlrpc_reply_state, + rs_list); + cfs_list_del(&rs->rs_list); + OBD_FREE_LARGE(rs, svc->srv_max_reply_size); + } + } while (0); +} - cfs_list_for_each_entry_safe(rs, t, &svcpt->scp_rep_idle, rs_list) { - cfs_list_del(&rs->rs_list); - OBD_FREE_LARGE(rs, service->srv_max_reply_size); - } +static void +ptlrpc_service_free(struct ptlrpc_service *svc) +{ + struct ptlrpc_service_part *svcpt; + struct ptlrpc_at_array *array; - /* In case somebody rearmed this in the meantime */ - cfs_timer_disarm(&svcpt->scp_at_timer); + do { /* iterrate over multiple partitions in the future */ + svcpt = svc->srv_part; + if (svcpt == NULL || svcpt->scp_service == NULL) + break; - array = &svcpt->scp_at_array; - if (array->paa_reqs_array != NULL) { - OBD_FREE(array->paa_reqs_array, - sizeof(cfs_list_t) * array->paa_size); - array->paa_reqs_array = NULL; - } + /* In case somebody rearmed this in the meantime */ + cfs_timer_disarm(&svcpt->scp_at_timer); + array = &svcpt->scp_at_array; - if (array->paa_reqs_count != NULL) { - OBD_FREE(array->paa_reqs_count, - sizeof(__u32) * array->paa_size); - array->paa_reqs_count= NULL; - } + if (array->paa_reqs_array != NULL) { + OBD_FREE(array->paa_reqs_array, + sizeof(cfs_list_t) * array->paa_size); + array->paa_reqs_array = NULL; + } + + if (array->paa_reqs_count != NULL) { + OBD_FREE(array->paa_reqs_count, + sizeof(__u32) * array->paa_size); + array->paa_reqs_count = NULL; + } + svcpt->scp_service = NULL; + } while (0); + + do { /* iterrate over multiple partitions in the future */ + svcpt = svc->srv_part; + if (svcpt != NULL) + OBD_FREE_PTR(svcpt); + } while (0); + + OBD_FREE_PTR(svc); +} + +int ptlrpc_unregister_service(struct ptlrpc_service *service) +{ + ENTRY; + + CDEBUG(D_NET, "%s: tearing down\n", service->srv_name); + + service->srv_is_stopping = 1; + + cfs_spin_lock(&ptlrpc_all_services_lock); + cfs_list_del_init(&service->srv_list); + cfs_spin_unlock(&ptlrpc_all_services_lock); + + ptlrpc_lprocfs_unregister_service(service); + + ptlrpc_service_del_atimer(service); + ptlrpc_stop_all_threads(service); + + ptlrpc_service_unlink_rqbd(service); + ptlrpc_service_purge_all(service); + ptlrpc_service_free(service); - OBD_FREE_PTR(svcpt); - out: - OBD_FREE_PTR(service); RETURN(0); }