From 2118a8b92cec2df85d1bdbe2e58b389d83fe06b2 Mon Sep 17 00:00:00 2001 From: Liang Zhen Date: Tue, 29 May 2012 15:34:13 +0800 Subject: [PATCH] LU-56 lnet: eliminate a few locking dance in LNet This patch gets rid of a few unnecessary locking dance from LNet, also changed type of lnet_ni_t::ni_last_alive from jiffy to second because per-second checking should be good enough. Signed-off-by: Liang Zhen Change-Id: I0e3269f0a5ae0cfecd9611584d5e213a670e38f6 Reviewed-on: http://review.whamcloud.com/2933 Tested-by: Hudson Reviewed-by: Doug Oucharek Tested-by: Maloo Reviewed-by: Bobi Jam Reviewed-by: Andreas Dilger --- lnet/include/lnet/lib-types.h | 8 ++++--- lnet/lnet/api-ni.c | 2 +- lnet/lnet/lib-move.c | 19 +++++++-------- lnet/lnet/router.c | 55 ++++++++++++++++++++----------------------- 4 files changed, 40 insertions(+), 44 deletions(-) diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index 8febd47..2e5133a 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -401,9 +401,11 @@ typedef struct lnet_ni { void *ni_data; /* instance-specific data */ lnd_t *ni_lnd; /* procedural interface */ int ni_refcount; /* reference count */ - cfs_time_t ni_last_alive; /* when I was last alive */ - lnet_ni_status_t *ni_status; /* my health status */ - char *ni_interfaces[LNET_MAX_INTERFACES]; /* equivalent interfaces to use */ + /* when I was last alive */ + long ni_last_alive; + lnet_ni_status_t *ni_status; /* my health status */ + /* equivalent interfaces to use */ + char *ni_interfaces[LNET_MAX_INTERFACES]; } lnet_ni_t; #define LNET_PROTO_PING_MATCHBITS 0x8000000000000000LL diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 4912be7..f2b4b2b 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -1182,7 +1182,7 @@ LNetNIInit(lnet_pid_t requested_pid) /* Now I may use my own API functions... */ /* NB router checker needs the_lnet.ln_ping_info in - * lnet_router_checker -> lnet_update_ni_status */ + * lnet_router_checker -> lnet_update_ni_status_locked */ rc = lnet_ping_target_init(); if (rc != 0) goto failed3; diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index c760a92..96fb6db 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -1854,17 +1854,16 @@ lnet_parse(lnet_ni_t *ni, lnet_hdr_t *hdr, lnet_nid_t from_nid, return -EPROTO; } - if (the_lnet.ln_routing) { - cfs_time_t now = cfs_time_current(); - - LNET_LOCK(); - - ni->ni_last_alive = now; - if (ni->ni_status != NULL && - ni->ni_status->ns_status == LNET_NI_STATUS_DOWN) - ni->ni_status->ns_status = LNET_NI_STATUS_UP; + if (the_lnet.ln_routing && + ni->ni_last_alive != cfs_time_current_sec()) { + LNET_LOCK(); - LNET_UNLOCK(); + /* NB: so far here is the only place to set NI status to "up */ + ni->ni_last_alive = cfs_time_current_sec(); + if (ni->ni_status != NULL && + ni->ni_status->ns_status == LNET_NI_STATUS_DOWN) + ni->ni_status->ns_status = LNET_NI_STATUS_UP; + LNET_UNLOCK(); } /* Regard a bad destination NID as a protocol error. Senders should diff --git a/lnet/lnet/router.c b/lnet/lnet/router.c index 08bb57c..86d24ad 100644 --- a/lnet/lnet/router.c +++ b/lnet/lnet/router.c @@ -736,40 +736,35 @@ lnet_wait_known_routerstate(void) } void -lnet_update_ni_status(void) +lnet_update_ni_status_locked(void) { - cfs_time_t now = cfs_time_current(); - lnet_ni_t *ni; - int status; - int timeout; - - LASSERT (the_lnet.ln_routing); + lnet_ni_t *ni; + long now; + int timeout; - timeout = router_ping_timeout + - MAX(live_router_check_interval, dead_router_check_interval); + LASSERT(the_lnet.ln_routing); - LNET_LOCK(); + timeout = router_ping_timeout + + MAX(live_router_check_interval, dead_router_check_interval); - cfs_list_for_each_entry (ni, &the_lnet.ln_nis, ni_list) { - lnet_ni_status_t *ns = ni->ni_status; + now = cfs_time_current_sec(); + cfs_list_for_each_entry(ni, &the_lnet.ln_nis, ni_list) { + if (ni->ni_lnd->lnd_type == LOLND) + continue; - LASSERT (ns != NULL); + if (now < ni->ni_last_alive + timeout) + continue; - status = LNET_NI_STATUS_UP; - if (ni->ni_lnd->lnd_type != LOLND && /* @lo forever alive */ - cfs_time_after(now, cfs_time_add(ni->ni_last_alive, - cfs_time_seconds(timeout)))) - status = LNET_NI_STATUS_DOWN; + LASSERT(ni->ni_status != NULL); - if (ns->ns_status != status) { - ns->ns_status = status; - CDEBUG(D_NET, "NI(%s:%d) status changed to %s\n", - libcfs_nid2str(ni->ni_nid), timeout, - status == LNET_NI_STATUS_UP ? "up" : "down"); - } - } - - LNET_UNLOCK(); + if (ni->ni_status->ns_status != LNET_NI_STATUS_DOWN) { + CDEBUG(D_NET, "NI(%s:%d) status changed to down\n", + libcfs_nid2str(ni->ni_nid), timeout); + /* NB: so far, this is the only place to set + * NI status to "down" */ + ni->ni_status->ns_status = LNET_NI_STATUS_DOWN; + } + } } void @@ -1196,10 +1191,10 @@ rescan: } } - LNET_UNLOCK(); + if (the_lnet.ln_routing) + lnet_update_ni_status_locked(); - if (the_lnet.ln_routing) - lnet_update_ni_status(); + LNET_UNLOCK(); lnet_prune_rc_data(0); /* don't wait for UNLINK */ -- 1.8.3.1