From 7eb61e89fcaeb4f2c036e6e19818e4217c207fa8 Mon Sep 17 00:00:00 2001 From: eeb Date: Fri, 2 Sep 2005 18:56:56 +0000 Subject: [PATCH] * Added hopcounts to route table * Changed ptlrpc_uuid_to_peer() to choose the matching UUID with the shortest hopcount * Changed lconf to use a single UUID string for all target NIDs, so the client can choose which one to use at runtime. * Stripped out all the unused network configuration stuff from lconf --- lnet/include/lnet/api.h | 1 + lnet/include/lnet/lib-lnet.h | 4 +- lnet/libcfs/nidstrings.c | 16 +++--- lnet/lnet/api-ni.c | 23 ++++++++ lnet/lnet/config.c | 34 +++++++++-- lnet/lnet/lib-move.c | 9 +++ lnet/lnet/module.c | 1 + lnet/lnet/router.c | 131 ++++++++++++++++++++++++++++++++----------- lnet/lnet/router.h | 5 +- lnet/lnet/router_proc.c | 12 +++- lnet/utils/portals.c | 33 ++++++++--- 11 files changed, 207 insertions(+), 62 deletions(-) diff --git a/lnet/include/lnet/api.h b/lnet/include/lnet/api.h index 709caaa..96e39a1 100644 --- a/lnet/include/lnet/api.h +++ b/lnet/include/lnet/api.h @@ -143,5 +143,6 @@ int LNetGet(lnet_handle_md_t md_in, lnet_match_bits_t match_bits_in, lnet_size_t offset_in); +int LNetDist (lnet_handle_ni_t interface, lnet_nid_t nid); #endif diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 84e0d28..a27ef3f 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -371,8 +371,9 @@ void lnet_fwd_done(ptl_ni_t *ni, kpr_fwd_desc_t *fwd, int error); int lnet_notify(ptl_ni_t *ni, lnet_nid_t peer, int alive, time_t when); /* internal APIs */ +int kpr_distance(lnet_nid_t nid); int kpr_ctl(unsigned int cmd, void *arg); -int kpr_add_route(__u32 net, lnet_nid_t gateway_nid); +int kpr_add_route(__u32 net, unsigned int hops, lnet_nid_t gateway_nid); int kpr_initialise(void); void kpr_finalise(void); @@ -439,6 +440,7 @@ extern int ptl_get_apinih (lnet_handle_ni_t *nih); extern ptl_ni_t *lnet_net2ni (__u32 net); extern int ptl_islocalnid (lnet_nid_t nid); +extern int ptl_islocalnet (__u32 net); extern void ptl_enq_event_locked (void *private, ptl_eq_t *eq, lnet_event_t *ev); extern void lnet_finalize (ptl_ni_t *ni, void *private, ptl_msg_t *msg, diff --git a/lnet/libcfs/nidstrings.c b/lnet/libcfs/nidstrings.c index 191f66c..9580fc6 100644 --- a/lnet/libcfs/nidstrings.c +++ b/lnet/libcfs/nidstrings.c @@ -100,42 +100,42 @@ struct nalstrfns { static struct nalstrfns libcfs_nalstrfns[] = { {.nf_nal = LONAL, .nf_name = "lo", - .nf_modname = "klonal", + .nf_modname = "klolnd", .nf_addr2str = libcfs_num_addr2str, .nf_str2addr = libcfs_lo_str2addr}, {.nf_nal = SOCKNAL, .nf_name = "tcp", - .nf_modname = "ksocknal", + .nf_modname = "ksocklnd", .nf_addr2str = libcfs_ip_addr2str, .nf_str2addr = libcfs_ip_str2addr}, {.nf_nal = OPENIBNAL, .nf_name = "openib", - .nf_modname = "kopenibnal", + .nf_modname = "kopeniblnd", .nf_addr2str = libcfs_ip_addr2str, .nf_str2addr = libcfs_ip_str2addr}, {.nf_nal = IIBNAL, .nf_name = "iib", - .nf_modname = "kiibnal", + .nf_modname = "kiiblnd", .nf_addr2str = libcfs_ip_addr2str, .nf_str2addr = libcfs_ip_str2addr}, {.nf_nal = VIBNAL, .nf_name = "vib", - .nf_modname = "kvibnal", + .nf_modname = "kviblnd", .nf_addr2str = libcfs_ip_addr2str, .nf_str2addr = libcfs_ip_str2addr}, {.nf_nal = RANAL, .nf_name = "ra", - .nf_modname = "kranal", + .nf_modname = "kralnd", .nf_addr2str = libcfs_ip_addr2str, .nf_str2addr = libcfs_ip_str2addr}, {.nf_nal = QSWNAL, .nf_name = "elan", - .nf_modname = "kqswnal", + .nf_modname = "kqswlnd", .nf_addr2str = libcfs_num_addr2str, .nf_str2addr = libcfs_num_str2addr}, {.nf_nal = GMNAL, .nf_name = "gm", - .nf_modname = "kgmnal", + .nf_modname = "kgmlnd", .nf_addr2str = libcfs_num_addr2str, .nf_str2addr = libcfs_num_str2addr}, }; diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index a999802..638c927 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -613,6 +613,29 @@ ptl_islocalnid (lnet_nid_t nid) return islocal; } +int +ptl_islocalnet (__u32 net) +{ + struct list_head *tmp; + ptl_ni_t *ni; + unsigned long flags; + int islocal = 0; + + PTL_LOCK(flags); + + list_for_each (tmp, &lnet_apini.apini_nis) { + ni = list_entry(tmp, ptl_ni_t, ni_list); + + if (PTL_NIDNET(ni->ni_nid) == net) { + islocal = 1; + break; + } + } + + PTL_UNLOCK(flags); + return islocal; +} + void ptl_shutdown_nalnis (void) { diff --git a/lnet/lnet/config.c b/lnet/lnet/config.c index aeff86b..38a014b 100644 --- a/lnet/lnet/config.c +++ b/lnet/lnet/config.c @@ -522,6 +522,18 @@ ptl_str2tbs_expand (struct list_head *tbs, char *str) } int +lnet_parse_hops (char *str, unsigned int *hops) +{ + int len = strlen(str); + int nob = len; + + return (sscanf(str, "%u%n", hops, &nob) >= 1 && + nob == len && + *hops > 0 && *hops < 256); +} + + +int lnet_parse_route (char *str) { /* static scratch buffer OK (single threaded) */ @@ -532,13 +544,15 @@ lnet_parse_route (char *str) struct list_head *tmp1; struct list_head *tmp2; __u32 net; - lnet_nid_t nid; + lnet_nid_t nid; ptl_text_buf_t *ptb; int rc; char *sep; char *token = str; int ntokens = 0; int myrc = -1; + unsigned int hops; + int got_hops = 0; INIT_LIST_HEAD(&gateways); INIT_LIST_HEAD(&nets); @@ -553,7 +567,7 @@ lnet_parse_route (char *str) while (ptl_iswhite(*sep)) sep++; if (*sep == 0) { - if (ntokens < 2) + if (ntokens < (got_hops ? 3 : 2)) goto token_error; break; } @@ -567,11 +581,16 @@ lnet_parse_route (char *str) if (*sep != 0) *sep++ = 0; - if (ntokens == 1) + if (ntokens == 1) { tmp2 = &nets; /* expanding nets */ - else + } else if (ntokens == 2 && + lnet_parse_hops(token, &hops)) { + got_hops = 1; /* got a hop count */ + continue; + } else { tmp2 = &gateways; /* expanding gateways */ - + } + ptb = ptl_new_text_buf(strlen(token)); if (ptb == NULL) goto out; @@ -607,6 +626,9 @@ lnet_parse_route (char *str) } } + if (!got_hops) + hops = 1; + LASSERT (!list_empty(&nets)); LASSERT (!list_empty(&gateways)); @@ -620,7 +642,7 @@ lnet_parse_route (char *str) nid = libcfs_str2nid(ptb->ptb_text); LASSERT (nid != LNET_NID_ANY); - rc = kpr_add_route (net, nid); + rc = kpr_add_route (net, hops, nid); if (rc != 0) { CERROR("Can't create route " "to %s via %s\n", diff --git a/lnet/lnet/lib-move.c b/lnet/lnet/lib-move.c index a2b9260..98cdacd 100644 --- a/lnet/lnet/lib-move.c +++ b/lnet/lnet/lib-move.c @@ -1370,3 +1370,12 @@ LNetGet(lnet_handle_md_t mdh, lnet_process_id_t target, /* completion will be signalled by an event */ return 0; } + +int +LNetDist (lnet_handle_ni_t interface, lnet_nid_t nid) +{ + LASSERT (lnet_apini.apini_init); + LASSERT (lnet_apini.apini_refcount > 0); + + return kpr_distance(nid); +} diff --git a/lnet/lnet/module.c b/lnet/lnet/module.c index e192d61..41e85a7 100644 --- a/lnet/lnet/module.c +++ b/lnet/lnet/module.c @@ -137,6 +137,7 @@ EXPORT_SYMBOL(LNetEQFree); EXPORT_SYMBOL(LNetEQGet); EXPORT_SYMBOL(LNetGetId); EXPORT_SYMBOL(LNetMDBind); +EXPORT_SYMBOL(LNetDist); EXPORT_SYMBOL(lnet_apini); EXPORT_SYMBOL(lnet_iov_nob); EXPORT_SYMBOL(lnet_copy_iov2buf); diff --git a/lnet/lnet/router.c b/lnet/lnet/router.c index e4ed00b..9a91a373 100644 --- a/lnet/lnet/router.c +++ b/lnet/lnet/router.c @@ -238,7 +238,6 @@ lnet_lookup (ptl_ni_t **nip, lnet_nid_t target_nid, int nob) int found; unsigned long flags; ptl_ni_t *gwni = NULL; - ptl_ni_t *tmpni = NULL; kpr_gateway_entry_t *ge = NULL; __u32 target_net = PTL_NIDNET(target_nid); @@ -272,7 +271,7 @@ lnet_lookup (ptl_ni_t **nip, lnet_nid_t target_nid, int nob) return LNET_NID_ANY; } - /* Search routes for one that has a gateway to target_nid on the callers network */ + /* Search for the target net */ found = 0; list_for_each (e, &kpr_state.kpr_nets) { ne = list_entry (e, kpr_net_entry_t, kpne_list); @@ -304,29 +303,21 @@ lnet_lookup (ptl_ni_t **nip, lnet_nid_t target_nid, int nob) kpr_ge_isbetter (ge, re->kpre_gateway)) continue; - } else if (gwni != NULL && - PTL_NIDNET(gwni->ni_nid) == - PTL_NIDNET(ge->kpge_nid)) { - /* another gateway on the same net */ + } else if (ge != NULL) { /* another gateway */ + LASSERT (gwni != NULL); + /* we don't allow gateways on different nets to get + * into the route table */ + LASSERT (PTL_NIDNET(gwni->ni_nid) == + PTL_NIDNET(re->kpre_gateway->kpge_nid)); if (kpr_ge_isbetter(ge, re->kpre_gateway)) continue; } else { - /* another gateway on a new/different net */ + LASSERT (gwni == NULL); - tmpni = lnet_net2ni(PTL_NIDNET(re->kpre_gateway->kpge_nid)); - if (tmpni == NULL) /* gateway not on a local net */ - continue; - - if (ge != NULL && - kpr_ge_isbetter(ge, re->kpre_gateway)) { - ptl_ni_decref(tmpni); + gwni = lnet_net2ni(PTL_NIDNET(re->kpre_gateway->kpge_nid)); + if (gwni == NULL) /* gateway not on a local net */ continue; - } - - if (gwni != NULL) - ptl_ni_decref(gwni); - gwni = tmpni; } ge = re->kpre_gateway; @@ -359,6 +350,33 @@ lnet_lookup (ptl_ni_t **nip, lnet_nid_t target_nid, int nob) return gwnid; } +int +kpr_distance (lnet_nid_t nid) +{ + unsigned long flags; + struct list_head *e; + kpr_net_entry_t *ne; + __u32 net = PTL_NIDNET(nid); + int dist = -ENETUNREACH; + + if (ptl_islocalnet(net)) + return 0; + + read_lock_irqsave(&kpr_state.kpr_rwlock, flags); + + list_for_each (e, &kpr_state.kpr_nets) { + ne = list_entry (e, kpr_net_entry_t, kpne_list); + + if (ne->kpne_net == net) { + dist = ne->kpne_hops; + break; + } + } + + read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); + return dist; +} + void lnet_fwd_start (ptl_ni_t *src_ni, kpr_fwd_desc_t *fwd) { @@ -579,7 +597,7 @@ lnet_fwd_done (ptl_ni_t *dst_ni, kpr_fwd_desc_t *fwd, int error) } int -kpr_add_route (__u32 net, lnet_nid_t gateway_nid) +kpr_add_route (__u32 net, unsigned int hops, lnet_nid_t gateway_nid) { unsigned long flags; struct list_head *e; @@ -588,10 +606,11 @@ kpr_add_route (__u32 net, lnet_nid_t gateway_nid) kpr_gateway_entry_t *ge = NULL; int dup = 0; - CDEBUG(D_NET, "Add route: net %s : gw %s\n", - libcfs_net2str(net), libcfs_nid2str(gateway_nid)); + CDEBUG(D_NET, "Add route: net %s hops %u gw %s\n", + libcfs_net2str(net), hops, libcfs_nid2str(gateway_nid)); - if (gateway_nid == LNET_NID_ANY) + if (gateway_nid == LNET_NID_ANY || + hops < 1 || hops > 255) return (-EINVAL); /* Assume net, route, gateway all new */ @@ -616,6 +635,7 @@ kpr_add_route (__u32 net, lnet_nid_t gateway_nid) atomic_set (&ge->kpge_weight, 0); ne->kpne_net = net; + ne->kpne_hops = hops; INIT_LIST_HEAD(&ne->kpne_routes); LASSERT(!in_interrupt()); @@ -636,22 +656,45 @@ kpr_add_route (__u32 net, lnet_nid_t gateway_nid) if (!dup) { /* Adding a new network? */ list_add_tail(&ne->kpne_list, &kpr_state.kpr_nets); } else { + if (ne->kpne_hops != hops) { + unsigned int hops2 = ne->kpne_hops; + + write_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); + + CERROR("Inconsistent hop count %d(%d) for network %s\n", + hops, hops2, libcfs_net2str(net)); + PORTAL_FREE(re, sizeof(*re)); + PORTAL_FREE(ge, sizeof(*ge)); + return -EINVAL; + } + dup = 0; list_for_each (e, &ne->kpne_routes) { kpr_route_entry_t *re2 = list_entry(e, kpr_route_entry_t, kpre_list); + + if (PTL_NIDNET(re2->kpre_gateway->kpge_nid) != + PTL_NIDNET(gateway_nid)) { + /* different gateway nets is an error */ + dup = -1; + break; + } - dup = (re2->kpre_gateway->kpge_nid == gateway_nid); - if (dup) + if (re2->kpre_gateway->kpge_nid == gateway_nid) { + /* same gateway is a noop */ + dup = 1; break; + } } - if (dup) { /* Ignore duplicate route entry */ + if (dup != 0) { + /* Don't add duplicate/bad route entry */ write_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); PORTAL_FREE(re, sizeof(*re)); PORTAL_FREE(ge, sizeof(*ge)); - return 0; + + return (dup < 0) ? -EINVAL : 0; } } @@ -754,7 +797,8 @@ kpr_del_route (__u32 net, lnet_nid_t gw_nid) } int -kpr_get_route (int idx, __u32 *net, lnet_nid_t *gateway_nid, __u32 *alive) +kpr_get_route (int idx, __u32 *net, __u32 *hops, + lnet_nid_t *gateway_nid, __u32 *alive, __u32 *ignored) { struct list_head *e1; struct list_head *e2; @@ -775,8 +819,11 @@ kpr_get_route (int idx, __u32 *net, lnet_nid_t *gateway_nid, __u32 *alive) if (idx-- == 0) { *net = ne->kpne_net; + *hops = ne->kpne_hops; *gateway_nid = ge->kpge_nid; *alive = ge->kpge_alive; + *ignored = !ptl_islocalnet(ne->kpne_net) && + ptl_islocalnet(ge->kpge_nid); read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); @@ -799,15 +846,25 @@ kpr_ctl(unsigned int cmd, void *arg) return -EINVAL; case IOC_PORTAL_ADD_ROUTE: - return kpr_add_route(data->ioc_net, data->ioc_nid); + return kpr_add_route(data->ioc_net, data->ioc_count, + data->ioc_nid); case IOC_PORTAL_DEL_ROUTE: return kpr_del_route (data->ioc_net, data->ioc_nid); - case IOC_PORTAL_GET_ROUTE: - return kpr_get_route(data->ioc_count, &data->ioc_net, - &data->ioc_nid, &data->ioc_flags); - + case IOC_PORTAL_GET_ROUTE: { + int alive; + int ignored; + int rc; + + rc = kpr_get_route(data->ioc_count, + &data->ioc_net, &data->ioc_count, + &data->ioc_nid, &alive, &ignored); + data->ioc_flags = ( alive ? 1 : 0) | + (ignored ? 2 : 0); + return rc; + } + case IOC_PORTAL_NOTIFY_ROUTER: return lnet_notify(NULL, data->ioc_nid, data->ioc_flags, (time_t)data->ioc_u64[0]); @@ -911,7 +968,7 @@ lnet_lookup (ptl_ni_t **nip, lnet_nid_t target_nid, int nob) } int -kpr_add_route (__u32 net, lnet_nid_t gateway_nid) +kpr_add_route (__u32 net, int hops, lnet_nid_t gateway_nid) { return -EOPNOTSUPP; } @@ -922,6 +979,12 @@ kpr_ctl(unsigned int cmd, void *arg) return -EINVAL; } +int +kpr_distance(lnet_nid_t nid) +{ + return 0; +} + void kpr_finalise (void) { diff --git a/lnet/lnet/router.h b/lnet/lnet/router.h index cb7f29d..7fb71cd 100644 --- a/lnet/lnet/router.h +++ b/lnet/lnet/router.h @@ -37,7 +37,7 @@ typedef struct time_t kpge_timestamp; int kpge_alive; int kpge_refcount; - lnet_nid_t kpge_nid; + lnet_nid_t kpge_nid; } kpr_gateway_entry_t; typedef struct @@ -51,12 +51,13 @@ typedef struct struct list_head kpne_list; struct list_head kpne_routes; __u32 kpne_net; + unsigned int kpne_hops; } kpr_net_entry_t; typedef struct { work_struct_t kpru_tq; - lnet_nid_t kpru_nid; + lnet_nid_t kpru_nid; int kpru_alive; time_t kpru_when; } kpr_upcall_t; diff --git a/lnet/lnet/router_proc.c b/lnet/lnet/router_proc.c index 8ae1964..d696772 100644 --- a/lnet/lnet/router_proc.c +++ b/lnet/lnet/router_proc.c @@ -197,8 +197,10 @@ kpr_seq_routes_show (struct seq_file *s, void *iter) kpr_seq_route_iterator_t *sri = iter; unsigned long flags; __u32 net; - lnet_nid_t nid; + unsigned int hops; + lnet_nid_t nid; int alive; + int ignored; read_lock_irqsave(&kpr_state.kpr_rwlock, flags); @@ -211,13 +213,17 @@ kpr_seq_routes_show (struct seq_file *s, void *iter) } net = sri->sri_net->kpne_net; + hops = sri->sri_net->kpne_hops; nid = sri->sri_route->kpre_gateway->kpge_nid; alive = sri->sri_route->kpre_gateway->kpge_alive; + ignored = !ptl_islocalnet(sri->sri_net->kpne_net) && + ptl_islocalnet(sri->sri_route->kpre_gateway->kpge_nid); read_unlock_irqrestore(&kpr_state.kpr_rwlock, flags); - seq_printf(s, "%-8s %4s %s\n", - libcfs_net2str(net), + seq_printf(s, "%-8s %2u %7s %s\n", + libcfs_net2str(net), hops, + ignored ? "ignored" : alive ? "up" : "down", libcfs_nid2str(nid)); return 0; diff --git a/lnet/utils/portals.c b/lnet/utils/portals.c index 2e4c9fd..9cc96f9 100644 --- a/lnet/utils/portals.c +++ b/lnet/utils/portals.c @@ -994,12 +994,14 @@ int jt_ptl_add_route (int argc, char **argv) { struct portal_ioctl_data data; - lnet_nid_t gateway_nid; + lnet_nid_t gateway_nid; + unsigned int hops = 1; + char *end; int rc; - if (argc != 2) + if (argc < 2 || argc > 3) { - fprintf (stderr, "usage: %s gateway\n", argv[0]); + fprintf (stderr, "usage: %s gateway [hopcount]\n", argv[0]); return (0); } @@ -1012,8 +1014,17 @@ jt_ptl_add_route (int argc, char **argv) return (-1); } + if (argc == 3) { + hops = strtoul(argv[2], &end, 0); + if (hops >= 256 || *end != 0) { + fprintf (stderr, "Can't parse hopcount \"%s\"\n", argv[2]); + return -1; + } + } + PORTAL_IOC_INIT(data); data.ioc_net = g_net; + data.ioc_count = hops; data.ioc_nid = gateway_nid; rc = l_ioctl(LNET_DEV_ID, IOC_PORTAL_ADD_ROUTE, &data); @@ -1124,7 +1135,9 @@ jt_ptl_print_routes (int argc, char **argv) int index; __u32 net; lnet_nid_t nid; + unsigned int hops; int alive; + int ignored; for (index = 0;;index++) { @@ -1135,12 +1148,16 @@ jt_ptl_print_routes (int argc, char **argv) if (rc != 0) break; - net = data.ioc_net; - nid = data.ioc_nid; - alive = data.ioc_flags; + net = data.ioc_net; + hops = data.ioc_count; + nid = data.ioc_nid; + alive = data.ioc_flags & 1; + ignored = data.ioc_flags & 2; - printf ("net %18s gw %32s %s\n", - libcfs_net2str(net), libcfs_nid2str(nid), + printf ("net %18s hops %u gw %32s %s\n", + libcfs_net2str(net), hops, + libcfs_nid2str(nid), + ignored ? "" : alive ? "up" : "down"); } -- 1.8.3.1