X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lnet%2Flnet%2Frouter.c;h=42df3a2ae8be8171a425fe5228ebf908bce08057;hb=ddea41245e375222fd272d32a77b4f31ad522a96;hp=938a4d6c7f9be406115a2da64cda08fc484b49ba;hpb=d7b4e4a631d3aead75f8beea07f2c129ea5765b9;p=fs%2Flustre-release.git diff --git a/lnet/lnet/router.c b/lnet/lnet/router.c index 938a4d6..42df3a2 100644 --- a/lnet/lnet/router.c +++ b/lnet/lnet/router.c @@ -1,7 +1,7 @@ /* -*- mode: c; c-basic-offset: 8; indent-tabs-mode: nil; -*- * vim:expandtab:shiftwidth=8:tabstop=8: * - * Copyright (C) 2002 Cluster File Systems, Inc. + * Copyright 2008 Sun Microsystems, Inc. All rights reserved * * This file is part of Portals * http://sourceforge.net/projects/sandiaportals/ @@ -39,6 +39,9 @@ CFS_MODULE_PARM(small_router_buffers, "i", int, 0444, static int large_router_buffers = 512; CFS_MODULE_PARM(large_router_buffers, "i", int, 0444, "# of large messages to buffer in the router"); +static int peer_buffer_credits = 0; +CFS_MODULE_PARM(peer_buffer_credits, "i", int, 0444, + "# router buffer credits per peer"); static int auto_down = 1; CFS_MODULE_PARM(auto_down, "i", int, 0444, @@ -66,6 +69,20 @@ lnet_peers_start_down(void) return check_routers_before_use; } +int +lnet_peer_buffer_credits(lnet_ni_t *ni) +{ + /* NI option overrides LNet default */ + if (ni->ni_peerrtrcredits > 0) + return ni->ni_peerrtrcredits; + if (peer_buffer_credits > 0) + return peer_buffer_credits; + + /* As an approximation, allow this peer the same number of router + * buffers as it is allowed outstanding sends */ + return ni->ni_peertxcredits; +} + void lnet_notify_locked(lnet_peer_t *lp, int notifylnd, int alive, time_t when) { @@ -94,14 +111,14 @@ lnet_notify_locked(lnet_peer_t *lp, int notifylnd, int alive, time_t when) } void -lnet_do_notify (lnet_peer_t *lp) +lnet_do_notify (lnet_peer_t *lp) { lnet_ni_t *ni = lp->lp_ni; int alive; int notifylnd; - + LNET_LOCK(); - + /* Notify only in 1 thread at any time to ensure ordered notification. * NB individual events can be missed; the only guarantee is that you * always get the most recent news */ @@ -112,7 +129,7 @@ lnet_do_notify (lnet_peer_t *lp) } lp->lp_notifying = 1; - + while (lp->lp_notify) { alive = lp->lp_alive; notifylnd = lp->lp_notifylnd; @@ -173,7 +190,7 @@ lnet_notify (lnet_ni_t *ni, lnet_nid_t nid, int alive, time_t when) CDEBUG(D_NET, "Auto-down disabled\n"); return 0; } - + LNET_LOCK(); lp = lnet_find_peer_locked(nid); @@ -184,12 +201,19 @@ lnet_notify (lnet_ni_t *ni, lnet_nid_t nid, int alive, time_t when) return 0; } + /* We can't fully trust LND on reporting exact peer last_alive + * if he notifies us about dead peer. For example ksocklnd can + * call us with when == _time_when_the_node_was_booted_ if + * no connections were successfully established */ + if (ni != NULL && !alive && when < lp->lp_last_alive) + when = lp->lp_last_alive; + lnet_notify_locked(lp, ni == NULL, alive, when); LNET_UNLOCK(); - + lnet_do_notify(lp); - + LNET_LOCK(); lnet_peer_decref_locked(lp); @@ -207,6 +231,12 @@ lnet_notify (lnet_ni_t *ni, lnet_nid_t nid, int alive, time_t when) return -EOPNOTSUPP; } +void +lnet_notify_locked (lnet_peer_t *lp, int notifylnd, int alive, time_t when) +{ + return; +} + #endif static void @@ -427,7 +457,7 @@ lnet_check_routes (void) else if (route->lr_gateway->lp_ni != route2->lr_gateway->lp_ni) { LNET_UNLOCK(); - + CERROR("Routes to %s via %s and %s not supported\n", libcfs_net2str(rnet->lrn_net), libcfs_nid2str(route->lr_gateway->lp_nid), @@ -436,7 +466,7 @@ lnet_check_routes (void) } } } - + LNET_UNLOCK(); return 0; } @@ -551,14 +581,14 @@ lnet_router_checker_event (lnet_event_t *event) /* The router checker thread has unlinked the rc_md * and exited. */ LASSERT (the_lnet.ln_rc_state == LNET_RC_STATE_UNLINKING); - the_lnet.ln_rc_state = LNET_RC_STATE_UNLINKED; - mutex_up(&the_lnet.ln_rc_signal); + the_lnet.ln_rc_state = LNET_RC_STATE_UNLINKED; + mutex_up(&the_lnet.ln_rc_signal); return; } - LASSERT (event->type == LNET_EVENT_SEND || + LASSERT (event->type == LNET_EVENT_SEND || event->type == LNET_EVENT_REPLY); - + nid = (event->type == LNET_EVENT_SEND) ? event->target.nid : event->initiator.nid; @@ -575,7 +605,7 @@ lnet_router_checker_event (lnet_event_t *event) if (lnet_isrouter(lp) && /* ignore if no longer a router */ (event->status != 0 || event->type == LNET_EVENT_REPLY)) { - + /* A successful REPLY means the router is up. If _any_ comms * to the router fail I assume it's down (this will happen if * we ping alive routers to try to detect router death before @@ -606,25 +636,29 @@ lnet_router_checker(void *arg) int rc; lnet_handle_md_t mdh; lnet_peer_t *rtr; + lnet_md_t md = {0}; struct list_head *entry; time_t now; lnet_process_id_t rtr_id; int secs; - cfs_daemonize("router_checker"); - cfs_block_allsigs(); + cfs_daemonize("router_checker"); + cfs_block_allsigs(); rtr_id.pid = LUSTRE_SRV_LNET_PID; LASSERT (the_lnet.ln_rc_state == LNET_RC_STATE_SHUTDOWN); - rc = LNetMDBind((lnet_md_t){.start = &pinginfo, - .length = sizeof(pinginfo), - .threshold = LNET_MD_THRESH_INF, - .options = LNET_MD_TRUNCATE, - .eq_handle = the_lnet.ln_rc_eqh}, - LNET_UNLINK, - &mdh); + /* initialize md content */ + md.start = &pinginfo; + md.length = sizeof(pinginfo); + md.threshold = LNET_MD_THRESH_INF; + md.max_size = 0; + md.options = LNET_MD_TRUNCATE, + md.user_ptr = NULL; + md.eq_handle = the_lnet.ln_rc_eqh; + + rc = LNetMDBind(md, LNET_UNLINK, &mdh); if (rc < 0) { CERROR("Can't bind MD: %d\n", rc); @@ -638,7 +672,7 @@ lnet_router_checker(void *arg) the_lnet.ln_rc_state = LNET_RC_STATE_RUNNING; mutex_up(&the_lnet.ln_rc_signal); /* let my parent go */ - while (the_lnet.ln_rc_state == LNET_RC_STATE_RUNNING) { + while (the_lnet.ln_rc_state == LNET_RC_STATE_RUNNING) { __u64 version; LNET_LOCK(); @@ -668,7 +702,7 @@ rescan: } if (secs <= 0) secs = 0; - + if (secs != 0 && !rtr->lp_ping_notsent && now > rtr->lp_ping_timestamp + secs) { @@ -690,7 +724,7 @@ rescan: LNET_RESERVED_PORTAL, LNET_PROTO_PING_MATCHBITS, 0); } - + LNET_LOCK(); lnet_peer_decref_locked(rtr); @@ -705,20 +739,18 @@ rescan: /* Call cfs_pause() here always adds 1 to load average * because kernel counts # active tasks as nr_running * + nr_uninterruptible. */ - set_current_state(CFS_TASK_INTERRUPTIBLE); cfs_schedule_timeout(CFS_TASK_INTERRUPTIBLE, cfs_time_seconds(1)); - } + } LASSERT (the_lnet.ln_rc_state == LNET_RC_STATE_STOPTHREAD); the_lnet.ln_rc_state = LNET_RC_STATE_UNLINKING; - + rc = LNetMDUnlink(mdh); LASSERT (rc == 0); /* The unlink event callback will signal final completion */ - - return 0; + return 0; } @@ -731,11 +763,11 @@ lnet_wait_known_routerstate(void) for (;;) { LNET_LOCK(); - + all_known = 1; list_for_each (entry, &the_lnet.ln_routers) { rtr = list_entry(entry, lnet_peer_t, lp_rtr_list); - + if (rtr->lp_alive_count == 0) { all_known = 0; break; @@ -763,14 +795,14 @@ lnet_router_checker_stop(void) return; the_lnet.ln_rc_state = LNET_RC_STATE_STOPTHREAD; - /* block until event callback signals exit */ - mutex_down(&the_lnet.ln_rc_signal); + /* block until event callback signals exit */ + mutex_down(&the_lnet.ln_rc_signal); LASSERT (the_lnet.ln_rc_state == LNET_RC_STATE_UNLINKED); rc = LNetEQFree(the_lnet.ln_rc_eqh); LASSERT (rc == 0); - + the_lnet.ln_rc_state = LNET_RC_STATE_SHUTDOWN; } @@ -788,12 +820,12 @@ lnet_router_checker_start(void) "\n"); return -EINVAL; } - + if (live_router_check_interval <= 0 && dead_router_check_interval <= 0) return 0; - init_mutex_locked(&the_lnet.ln_rc_signal); + init_mutex_locked(&the_lnet.ln_rc_signal); /* EQ size doesn't matter; the callback is guaranteed to get every * event */ @@ -804,20 +836,20 @@ lnet_router_checker_start(void) return -ENOMEM; } - rc = (int)cfs_kernel_thread(lnet_router_checker, NULL, 0); - if (rc < 0) { - CERROR("Can't start router checker thread: %d\n", rc); + rc = (int)cfs_kernel_thread(lnet_router_checker, NULL, 0); + if (rc < 0) { + CERROR("Can't start router checker thread: %d\n", rc); goto failed; - } + } - mutex_down(&the_lnet.ln_rc_signal); /* wait for checker to startup */ + mutex_down(&the_lnet.ln_rc_signal); /* wait for checker to startup */ rc = the_lnet.ln_rc_state; if (rc < 0) { the_lnet.ln_rc_state = LNET_RC_STATE_SHUTDOWN; goto failed; } - + LASSERT (the_lnet.ln_rc_state == LNET_RC_STATE_RUNNING); if (check_routers_before_use) { @@ -826,9 +858,9 @@ lnet_router_checker_start(void) * may have to a previous instance of me. */ lnet_wait_known_routerstate(); } - + return 0; - + failed: rc = LNetEQFree(the_lnet.ln_rc_eqh); LASSERT (rc == 0); @@ -915,7 +947,7 @@ lnet_rtrpool_alloc_bufs(lnet_rtrbufpool_t *rbp, int nbufs) LASSERT (rbp->rbp_nbuffers == nbufs); return 0; } - + for (i = 0; i < nbufs; i++) { rb = lnet_new_rtrbuf(rbp); @@ -974,7 +1006,7 @@ int lnet_alloc_rtrpools(int im_a_router) { int rc; - + if (!strcmp(forwarding, "")) { /* not set either way */ if (!im_a_router) @@ -989,7 +1021,7 @@ lnet_alloc_rtrpools(int im_a_router) "'enabled' or 'disabled'\n"); return -EINVAL; } - + if (tiny_router_buffers <= 0) { LCONSOLE_ERROR_MSG(0x10c, "tiny_router_buffers=%d invalid when " "routing enabled\n", tiny_router_buffers); @@ -1029,7 +1061,7 @@ lnet_alloc_rtrpools(int im_a_router) LNET_LOCK(); the_lnet.ln_routing = 1; LNET_UNLOCK(); - + return 0; failed: @@ -1045,6 +1077,12 @@ lnet_peers_start_down(void) return 0; } +int +lnet_peer_buffer_credits(lnet_ni_t *ni) +{ + return 0; +} + void lnet_router_checker_stop(void) {