From: isaac Date: Tue, 1 Dec 2009 15:05:28 +0000 (+0000) Subject: i=liang,b=15332,b=21103: X-Git-Tag: GIT_EPOCH_B_HD_KDMU~42 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=14a611ad680e89523abbcab0a3310511ab808ba8 i=liang,b=15332,b=21103: - LNet fine grain routing support. --- diff --git a/lnet/ChangeLog b/lnet/ChangeLog index 8f7a42d..ae041fa 100644 --- a/lnet/ChangeLog +++ b/lnet/ChangeLog @@ -17,6 +17,10 @@ Bugzilla : Description: Details : +Severity : enhancement +Bugzilla : 15332 +Description: LNet fine grain routing support. + Severity : normal Bugzilla : 20171 Description: router checker stops working when system wall clock goes backward diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index 9efa5da..4ae355c 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -454,13 +454,13 @@ typedef struct lnet_peer { typedef struct { struct list_head lr_list; /* chain on net */ lnet_peer_t *lr_gateway; /* router node */ + unsigned int lr_hops; /* how far I am */ } lnet_route_t; typedef struct { struct list_head lrn_list; /* chain on ln_remote_nets */ struct list_head lrn_routes; /* routes to me */ __u32 lrn_net; /* my net number */ - unsigned int lrn_hops; /* how far I am */ } lnet_remotenet_t; typedef struct { diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index 743146c..156a09a 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -791,8 +791,17 @@ lnet_ni_recv(lnet_ni_t *ni, void *private, lnet_msg_t *msg, int delayed, } int -lnet_compare_routers(lnet_peer_t *p1, lnet_peer_t *p2) +lnet_compare_routes(lnet_route_t *r1, lnet_route_t *r2) { + lnet_peer_t *p1 = r1->lr_gateway; + lnet_peer_t *p2 = r2->lr_gateway; + + if (r1->lr_hops < r2->lr_hops) + return 1; + + if (r1->lr_hops > r2->lr_hops) + return -1; + if (p1->lp_txqnob < p2->lp_txqnob) return 1; @@ -1340,7 +1349,7 @@ lnet_send(lnet_nid_t src_nid, lnet_msg_t *msg) lnet_ni_decref_locked(local_ni); lnet_ni_decref_locked(src_ni); LNET_UNLOCK(); - CERROR("no route to %s via from %s\n", + CERROR("No route to %s via from %s\n", libcfs_nid2str(dst_nid), libcfs_nid2str(src_nid)); return -EINVAL; } @@ -1402,7 +1411,8 @@ lnet_send(lnet_nid_t src_nid, lnet_msg_t *msg) if (lp2->lp_alive && lnet_router_down_ni(lp2, rnet->lrn_net) <= 0 && (src_ni == NULL || lp2->lp_ni == src_ni) && - (lp == NULL || lnet_compare_routers(lp2, lp) > 0)) { + (lp == NULL || + lnet_compare_routes(route, best_route) > 0)) { best_route = route; lp = lp2; } @@ -1412,8 +1422,10 @@ lnet_send(lnet_nid_t src_nid, lnet_msg_t *msg) if (src_ni != NULL) lnet_ni_decref_locked(src_ni); LNET_UNLOCK(); - CERROR("No route to %s (all routers down)\n", - libcfs_id2str(msg->msg_target)); + + CERROR("No route to %s via %s (all routers down)\n", + libcfs_id2str(msg->msg_target), + libcfs_nid2str(src_nid)); return -EHOSTUNREACH; } @@ -2619,7 +2631,6 @@ LNetDist (lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp) { struct list_head *e; lnet_ni_t *ni; - lnet_route_t *route; lnet_remotenet_t *rnet; __u32 dstnet = LNET_NIDNET(dstnid); int hops; @@ -2668,12 +2679,21 @@ LNetDist (lnet_nid_t dstnid, lnet_nid_t *srcnidp, __u32 *orderp) rnet = list_entry(e, lnet_remotenet_t, lrn_list); if (rnet->lrn_net == dstnet) { + lnet_route_t *route; + lnet_route_t *shortest = NULL; + LASSERT (!list_empty(&rnet->lrn_routes)); - route = list_entry(rnet->lrn_routes.next, - lnet_route_t, lr_list); - hops = rnet->lrn_hops; + + list_for_each_entry(route, &rnet->lrn_routes, lr_list) { + if (shortest == NULL || + route->lr_hops < shortest->lr_hops) + shortest = route; + } + + LASSERT (shortest != NULL); + hops = shortest->lr_hops; if (srcnidp != NULL) - *srcnidp = route->lr_gateway->lp_ni->ni_nid; + *srcnidp = shortest->lr_gateway->lp_ni->ni_nid; if (orderp != NULL) *orderp = order; LNET_UNLOCK(); diff --git a/lnet/lnet/peer.c b/lnet/lnet/peer.c index fd7d31c..2e7b9a4 100644 --- a/lnet/lnet/peer.c +++ b/lnet/lnet/peer.c @@ -227,8 +227,8 @@ lnet_nid2peer_locked(lnet_peer_t **lpp, lnet_nid_t nid) lp->lp_rtrcredits = lp->lp_minrtrcredits = lnet_peer_buffer_credits(lp->lp_ni); - LASSERT (!the_lnet.ln_shutdown); /* can't add peers after shutdown starts */ + LASSERT (!the_lnet.ln_shutdown); list_add_tail(&lp->lp_hashlist, lnet_nid2peerhash(nid)); the_lnet.ln_npeers++; diff --git a/lnet/lnet/router.c b/lnet/lnet/router.c index b89c5f3..a8f816c 100644 --- a/lnet/lnet/router.c +++ b/lnet/lnet/router.c @@ -238,12 +238,10 @@ lnet_find_net_locked (__u32 net) int lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) { - struct list_head zombies; struct list_head *e; lnet_remotenet_t *rnet; lnet_remotenet_t *rnet2; lnet_route_t *route; - lnet_route_t *route2; lnet_ni_t *ni; int add_route; int rc; @@ -277,7 +275,7 @@ lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) CFS_INIT_LIST_HEAD(&rnet->lrn_routes); rnet->lrn_net = net; - rnet->lrn_hops = hops; + route->lr_hops = hops; LNET_LOCK(); @@ -297,7 +295,6 @@ lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) } LASSERT (!the_lnet.ln_shutdown); - CFS_INIT_LIST_HEAD(&zombies); rnet2 = lnet_find_net_locked(net); if (rnet2 == NULL) { @@ -306,37 +303,24 @@ lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) rnet2 = rnet; } - if (hops > rnet2->lrn_hops) { - /* New route is longer; ignore it */ - add_route = 0; - } else if (hops < rnet2->lrn_hops) { - /* new route supercedes all currently known routes to this - * net */ - list_add(&zombies, &rnet2->lrn_routes); - list_del_init(&rnet2->lrn_routes); - add_route = 1; - } else { - add_route = 1; - /* New route has the same hopcount as existing routes; search - * for a duplicate route (it's a NOOP if it is) */ - list_for_each (e, &rnet2->lrn_routes) { - route2 = list_entry(e, lnet_route_t, lr_list); - - if (route2->lr_gateway == route->lr_gateway) { - add_route = 0; - break; - } + /* Search for a duplicate route (it's a NOOP if it is) */ + add_route = 1; + list_for_each (e, &rnet2->lrn_routes) { + lnet_route_t *route2 = list_entry(e, lnet_route_t, lr_list); - /* our loopups must be true */ - LASSERT (route2->lr_gateway->lp_nid != gateway); + if (route2->lr_gateway == route->lr_gateway) { + add_route = 0; + break; } + + /* our lookups must be true */ + LASSERT (route2->lr_gateway->lp_nid != gateway); } if (add_route) { ni = route->lr_gateway->lp_ni; lnet_ni_addref_locked(ni); - LASSERT (rc == 0); list_add_tail(&route->lr_list, &rnet2->lrn_routes); the_lnet.ln_remote_nets_version++; @@ -358,18 +342,7 @@ lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) if (rnet != rnet2) LIBCFS_FREE(rnet, sizeof(*rnet)); - while (!list_empty(&zombies)) { - route = list_entry(zombies.next, lnet_route_t, lr_list); - list_del(&route->lr_list); - - LNET_LOCK(); - lnet_rtr_decref_locked(route->lr_gateway); - lnet_peer_decref_locked(route->lr_gateway); - LNET_UNLOCK(); - LIBCFS_FREE(route, sizeof(*route)); - } - - return rc; + return 0; } int @@ -492,7 +465,7 @@ lnet_get_route (int idx, __u32 *net, __u32 *hops, if (idx-- == 0) { *net = rnet->lrn_net; - *hops = rnet->lrn_hops; + *hops = route->lr_hops; *gateway = route->lr_gateway->lp_nid; *alive = route->lr_gateway->lp_alive; LNET_UNLOCK(); diff --git a/lnet/lnet/router_proc.c b/lnet/lnet/router_proc.c index f04ccc9..fce91c2 100644 --- a/lnet/lnet/router_proc.c +++ b/lnet/lnet/router_proc.c @@ -179,7 +179,7 @@ int LL_PROC_PROTO(proc_lnet_routes) if (route != NULL) { __u32 net = rnet->lrn_net; - unsigned int hops = rnet->lrn_hops; + unsigned int hops = route->lr_hops; lnet_nid_t nid = route->lr_gateway->lp_nid; int alive = route->lr_gateway->lp_alive;