From 4443604ca437aaafe2a0dc700c1ceb8b131ba126 Mon Sep 17 00:00:00 2001 From: Doug Oucharek Date: Fri, 8 Mar 2013 10:10:22 -0800 Subject: [PATCH] LU-2934 lnet: Add LNet Router Priority parameter This change adds a priority parameter to the route module settings. This paramter can be >= 0. Like hops, the lower the prioirty number the higher the priority. So lower numbered priorities will be selected over higher numbers. Signed-off-by: Doug Oucharek Change-Id: Idca12ee44ec5e2a896f89c178334363ffdfef4b8 Reviewed-on: http://review.whamcloud.com/5663 Tested-by: Hudson Reviewed-by: Andreas Dilger Tested-by: Maloo Reviewed-by: Amir Shehata Reviewed-by: Isaac Huang Reviewed-by: Oleg Drokin --- libcfs/include/libcfs/libcfs_ioctl.h | 2 ++ lnet/include/lnet/lib-lnet.h | 5 ++-- lnet/include/lnet/lib-types.h | 2 +- lnet/lnet/api-ni.c | 7 +++-- lnet/lnet/config.c | 41 +++++++++++++++++++++++++-- lnet/lnet/lib-move.c | 6 ++++ lnet/lnet/router.c | 19 +++++++------ lnet/lnet/router_proc.c | 16 ++++++----- lnet/utils/portals.c | 54 +++++++++++++++++++++++------------- lustre/tests/sanity.sh | 9 +++--- lustre/utils/lctl.c | 2 +- 11 files changed, 115 insertions(+), 48 deletions(-) diff --git a/libcfs/include/libcfs/libcfs_ioctl.h b/libcfs/include/libcfs/libcfs_ioctl.h index 31c76ec..dcbae24 100644 --- a/libcfs/include/libcfs/libcfs_ioctl.h +++ b/libcfs/include/libcfs/libcfs_ioctl.h @@ -69,6 +69,8 @@ struct libcfs_ioctl_data { char ioc_bulk[0]; }; +#define ioc_priority ioc_u32[0] + struct libcfs_ioctl_hdr { __u32 ioc_len; diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 24d7aba..8f9710e 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -726,12 +726,13 @@ extern lnet_ni_t *lnet_net2ni(__u32 net); int lnet_notify(lnet_ni_t *ni, lnet_nid_t peer, int alive, cfs_time_t when); void lnet_notify_locked(lnet_peer_t *lp, int notifylnd, int alive, cfs_time_t when); -int lnet_add_route(__u32 net, unsigned int hops, lnet_nid_t gateway_nid); +int lnet_add_route(__u32 net, unsigned int hops, lnet_nid_t gateway_nid, + unsigned int priority); int lnet_check_routes(void); int lnet_del_route(__u32 net, lnet_nid_t gw_nid); void lnet_destroy_routes(void); int lnet_get_route(int idx, __u32 *net, __u32 *hops, - lnet_nid_t *gateway, __u32 *alive); + lnet_nid_t *gateway, __u32 *alive, __u32 *priority); void lnet_proc_init(void); void lnet_proc_fini(void); int lnet_rtrpools_alloc(int im_a_router); diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index bfaa288..8aaa239 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -505,7 +505,6 @@ typedef struct lnet_peer { lnet_rc_data_t *lp_rcd; /* router checker state */ } lnet_peer_t; - /* peer hash size */ #define LNET_PEER_HASH_BITS 9 #define LNET_PEER_HASH_SIZE (1 << LNET_PEER_HASH_BITS) @@ -531,6 +530,7 @@ typedef struct { int lr_seq; /* sequence for round-robin */ unsigned int lr_downis; /* number of down NIs */ unsigned int lr_hops; /* how far I am */ + unsigned int lr_priority; /* route priority */ } lnet_route_t; #define LNET_REMOTE_NETS_HASH_DEFAULT (1U << 7) diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 97bb5e7..ed5eba9 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -1599,8 +1599,8 @@ LNetCtl(unsigned int cmd, void *arg) return lnet_fail_nid(data->ioc_nid, data->ioc_count); case IOC_LIBCFS_ADD_ROUTE: - rc = lnet_add_route(data->ioc_net, data->ioc_count, - data->ioc_nid); + rc = lnet_add_route(data->ioc_net, data->ioc_count, + data->ioc_nid, data->ioc_priority); return (rc != 0) ? rc : lnet_check_routes(); case IOC_LIBCFS_DEL_ROUTE: @@ -1609,7 +1609,8 @@ LNetCtl(unsigned int cmd, void *arg) case IOC_LIBCFS_GET_ROUTE: return lnet_get_route(data->ioc_count, &data->ioc_net, &data->ioc_count, - &data->ioc_nid, &data->ioc_flags); + &data->ioc_nid, &data->ioc_flags, + &data->ioc_priority); case IOC_LIBCFS_NOTIFY_ROUTER: return lnet_notify(NULL, data->ioc_nid, data->ioc_flags, cfs_time_current() - diff --git a/lnet/lnet/config.c b/lnet/lnet/config.c index b18c103..deca792 100644 --- a/lnet/lnet/config.c +++ b/lnet/lnet/config.c @@ -614,6 +614,37 @@ lnet_parse_hops (char *str, unsigned int *hops) *hops > 0 && *hops < 256); } +#define LNET_PRIORITY_SEPARATOR (':') + +int +lnet_parse_priority(char *str, unsigned int *priority, char **token) +{ + int nob; + char *sep; + int len; + + sep = strchr(str, LNET_PRIORITY_SEPARATOR); + if (sep == NULL) { + *priority = 0; + return 0; + } + len = strlen(sep + 1); + + if ((sscanf((sep+1), "%u%n", priority, &nob) < 1) || (len != nob)) { + /* Update the caller's token pointer so it treats the found + priority as the token to report in the error message. */ + *token += sep - str + 1; + return -1; + } + + CDEBUG(D_NET, "gateway %s, priority %d, nob %d\n", str, *priority, nob); + + /* + * Change priority separator to \0 to be able to parse NID + */ + *sep = '\0'; + return 0; +} int lnet_parse_route (char *str, int *im_a_router) @@ -635,6 +666,7 @@ lnet_parse_route (char *str, int *im_a_router) int myrc = -1; unsigned int hops; int got_hops = 0; + unsigned int priority = 0; CFS_INIT_LIST_HEAD(&gateways); CFS_INIT_LIST_HEAD(&nets); @@ -702,6 +734,11 @@ lnet_parse_route (char *str, int *im_a_router) LNET_NETTYP(net) == LOLND) goto token_error; } else { + rc = lnet_parse_priority(ltb->ltb_text, + &priority, &token); + if (rc < 0) + goto token_error; + nid = libcfs_str2nid(ltb->ltb_text); if (nid == LNET_NID_ANY || LNET_NETTYP(LNET_NIDNET(nid)) == LOLND) @@ -724,14 +761,14 @@ lnet_parse_route (char *str, int *im_a_router) cfs_list_for_each (tmp2, &gateways) { ltb = cfs_list_entry(tmp2, lnet_text_buf_t, ltb_list); nid = libcfs_str2nid(ltb->ltb_text); - LASSERT (nid != LNET_NID_ANY); + LASSERT(nid != LNET_NID_ANY); if (lnet_islocalnid(nid)) { *im_a_router = 1; continue; } - rc = lnet_add_route (net, hops, nid); + rc = lnet_add_route(net, hops, nid, priority); if (rc != 0) { CERROR("Can't create route " "to %s via %s\n", diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index a531d94..0906690 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -1122,6 +1122,12 @@ lnet_compare_routes(lnet_route_t *r1, lnet_route_t *r2) lnet_peer_t *p1 = r1->lr_gateway; lnet_peer_t *p2 = r2->lr_gateway; + if (r1->lr_priority < r2->lr_priority) + return 1; + + if (r1->lr_priority > r2->lr_priority) + return -1; + if (r1->lr_hops < r2->lr_hops) return 1; diff --git a/lnet/lnet/router.c b/lnet/lnet/router.c index dd5e9e3..46adbeb 100644 --- a/lnet/lnet/router.c +++ b/lnet/lnet/router.c @@ -301,7 +301,8 @@ lnet_add_route_to_rnet (lnet_remotenet_t *rnet, lnet_route_t *route) } int -lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) +lnet_add_route(__u32 net, unsigned int hops, lnet_nid_t gateway, + unsigned int priority) { cfs_list_t *e; lnet_remotenet_t *rnet; @@ -311,8 +312,8 @@ lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) int add_route; int rc; - CDEBUG(D_NET, "Add route: net %s hops %u gw %s\n", - libcfs_net2str(net), hops, libcfs_nid2str(gateway)); + CDEBUG(D_NET, "Add route: net %s hops %u priority %u gw %s\n", + libcfs_net2str(net), hops, priority, libcfs_nid2str(gateway)); if (gateway == LNET_NID_ANY || LNET_NETTYP(LNET_NIDNET(gateway)) == LOLND || @@ -342,6 +343,7 @@ lnet_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway) rnet->lrn_net = net; route->lr_hops = hops; route->lr_net = net; + route->lr_priority = priority; lnet_net_lock(LNET_LOCK_EX); @@ -552,7 +554,7 @@ lnet_destroy_routes (void) int lnet_get_route(int idx, __u32 *net, __u32 *hops, - lnet_nid_t *gateway, __u32 *alive) + lnet_nid_t *gateway, __u32 *alive, __u32 *priority) { cfs_list_t *e1; cfs_list_t *e2; @@ -574,10 +576,11 @@ lnet_get_route(int idx, __u32 *net, __u32 *hops, lr_list); if (idx-- == 0) { - *net = rnet->lrn_net; - *hops = route->lr_hops; - *gateway = route->lr_gateway->lp_nid; - *alive = route->lr_gateway->lp_alive; + *net = rnet->lrn_net; + *hops = route->lr_hops; + *priority = route->lr_priority; + *gateway = route->lr_gateway->lp_nid; + *alive = route->lr_gateway->lp_alive; lnet_net_unlock(cpt); return 0; } diff --git a/lnet/lnet/router_proc.c b/lnet/lnet/router_proc.c index d4382b0..6bb052b 100644 --- a/lnet/lnet/router_proc.c +++ b/lnet/lnet/router_proc.c @@ -185,8 +185,8 @@ int LL_PROC_PROTO(proc_lnet_routes) the_lnet.ln_routing ? "enabled" : "disabled"); LASSERT (tmpstr + tmpsiz - s > 0); - s += snprintf(s, tmpstr + tmpsiz - s, "%-8s %4s %7s %s\n", - "net", "hops", "state", "router"); + s += snprintf(s, tmpstr + tmpsiz - s, "%-8s %4s %8s %7s %s\n", + "net", "hops", "priority", "state", "router"); LASSERT (tmpstr + tmpsiz - s > 0); lnet_net_lock(0); @@ -240,14 +240,16 @@ int LL_PROC_PROTO(proc_lnet_routes) } if (route != NULL) { - __u32 net = rnet->lrn_net; - unsigned int hops = route->lr_hops; - lnet_nid_t nid = route->lr_gateway->lp_nid; - int alive = route->lr_gateway->lp_alive; + __u32 net = rnet->lrn_net; + unsigned int hops = route->lr_hops; + unsigned int priority = route->lr_priority; + lnet_nid_t nid = route->lr_gateway->lp_nid; + int alive = route->lr_gateway->lp_alive; s += snprintf(s, tmpstr + tmpsiz - s, - "%-8s %4u %7s %s\n", + "%-8s %4u %8u %7s %s\n", libcfs_net2str(net), hops, + priority, alive ? "up" : "down", libcfs_nid2str(nid)); LASSERT(tmpstr + tmpsiz - s > 0); diff --git a/lnet/utils/portals.c b/lnet/utils/portals.c index 3c09a8d..2b3aada 100644 --- a/lnet/utils/portals.c +++ b/lnet/utils/portals.c @@ -1126,14 +1126,15 @@ jt_ptl_add_route (int argc, char **argv) struct libcfs_ioctl_data data; lnet_nid_t gateway_nid; unsigned int hops = 1; + unsigned int priority = 0; char *end; int rc; - if (argc < 2 || argc > 3) - { - fprintf (stderr, "usage: %s gateway [hopcount]\n", argv[0]); - return (0); - } + if (argc < 2 || argc > 4) { + fprintf(stderr, "usage: %s gateway [hopcount [priority]]\n", + argv[0]); + return -1; + } if (!g_net_is_set(argv[0])) return (-1); @@ -1144,18 +1145,29 @@ jt_ptl_add_route (int argc, char **argv) return (-1); } - if (argc == 3) { - hops = strtoul(argv[2], &end, 0); - if (hops >= 256 || *end != 0) { - fprintf (stderr, "Can't parse hopcount \"%s\"\n", argv[2]); - return -1; - } - } - - LIBCFS_IOC_INIT(data); - data.ioc_net = g_net; - data.ioc_count = hops; - data.ioc_nid = gateway_nid; + if (argc > 2) { + hops = strtoul(argv[2], &end, 0); + if (hops == 0 || hops >= 256 || (end != NULL && *end != 0)) { + fprintf(stderr, "Can't parse hopcount \"%s\"\n", + argv[2]); + return -1; + } + if (argc == 4) { + priority = strtoul(argv[3], &end, 0); + if (end != NULL && *end != 0) { + fprintf(stderr, + "Can't parse priority \"%s\"\n", + argv[3]); + return -1; + } + } + } + + LIBCFS_IOC_INIT(data); + data.ioc_net = g_net; + data.ioc_count = hops; + data.ioc_nid = gateway_nid; + data.ioc_priority = priority; rc = l_ioctl(LNET_DEV_ID, IOC_LIBCFS_ADD_ROUTE, &data); if (rc != 0) { @@ -1267,6 +1279,7 @@ jt_ptl_print_routes (int argc, char **argv) lnet_nid_t nid; unsigned int hops; int alive; + unsigned int pri; for (index = 0;;index++) { @@ -1281,10 +1294,11 @@ jt_ptl_print_routes (int argc, char **argv) hops = data.ioc_count; nid = data.ioc_nid; alive = data.ioc_flags; + pri = data.ioc_priority; - printf ("net %18s hops %u gw %32s %s\n", - libcfs_net2str(net), hops, - libcfs_nid2str(nid), alive ? "up" : "down"); + printf("net %18s hops %u gw %32s %s pri %u\n", + libcfs_net2str(net), hops, + libcfs_nid2str(nid), alive ? "up" : "down", pri); } if (errno != ENOENT) diff --git a/lustre/tests/sanity.sh b/lustre/tests/sanity.sh index 2b653bc..fcf5f72 100644 --- a/lustre/tests/sanity.sh +++ b/lustre/tests/sanity.sh @@ -10679,12 +10679,13 @@ test_215() { # for bugs 18102, 21079, 21517 # /proc/sys/lnet/routes should look like this: # Routing disabled/enabled - # net hops state router - # where net is a string like tcp0, hops >= 0, state is up/down, + # net hops priority state router + # where net is a string like tcp0, hops > 0, priority >= 0, + # state is up/down, # router is a string like 192.168.1.1@tcp2 L1="^Routing (disabled|enabled)$" - L2="^net +hops +state +router$" - BR="^$NET +$N +(up|down) +$NID$" + L2="^net +hops +priority +state +router$" + BR="^$NET +$N +(0|1) +(up|down) +$NID$" create_lnet_proc_files "routes" check_lnet_proc_entry "routes.out" "/proc/sys/lnet/routes" "$BR" "$L1" "$L2" check_lnet_proc_entry "routes.sys" "lnet.routes" "$BR" "$L1" "$L2" diff --git a/lustre/utils/lctl.c b/lustre/utils/lctl.c index b86a378..4363223 100644 --- a/lustre/utils/lctl.c +++ b/lustre/utils/lctl.c @@ -402,7 +402,7 @@ command_t cmdlist[] = { "usage: del_interface [ip]"}, {"add_route", jt_ptl_add_route, 0, "add an entry to the portals routing table\n" - "usage: add_route []"}, + "usage: add_route [ []]"}, {"del_route", jt_ptl_del_route, 0, "delete route via gateway to targets from the portals routing table\n" "usage: del_route [] []"}, -- 1.8.3.1