From: Serguei Smirnov Date: Tue, 16 Mar 2021 21:34:26 +0000 (-0400) Subject: LU-13641 socklnd: remove tcp bonding X-Git-Tag: 2.14.52~202 X-Git-Url: https://git.whamcloud.com/?p=fs%2Flustre-release.git;a=commitdiff_plain;h=d123c47a18adbf5665ed63d99c53117b84db9ec8;ds=sidebyside LU-13641 socklnd: remove tcp bonding TCP bonding in the socklnd has become obsolete with LNet Multi-Rail and there's no evidence it's being used anywhere. Remove it to keep the code simple. Test-Parameters: trivial testlist=sanity-lnet Signed-off-by: Serguei Smirnov Change-Id: Ib456f951b8ccd59112c460085632a2cb3c982004 Reviewed-on: https://review.whamcloud.com/40000 Reviewed-by: Andreas Dilger Tested-by: jenkins Tested-by: Maloo Reviewed-by: Chris Horn Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 1358f6f..176c9a2 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -866,8 +866,7 @@ void lnet_peer_push_event(struct lnet_event *ev); int lnet_parse_ip2nets(const char **networksp, const char *ip2nets); int lnet_parse_routes(const char *route_str, int *im_a_router); -int lnet_parse_networks(struct list_head *nilist, const char *networks, - bool use_tcp_bonding); +int lnet_parse_networks(struct list_head *nilist, const char *networks); bool lnet_net_unique(__u32 net_id, struct list_head *nilist, struct lnet_net **net); bool lnet_ni_unique_net(struct list_head *nilist, char *iface); diff --git a/lnet/include/lnet/lib-types.h b/lnet/include/lnet/lib-types.h index 010cf32..4ffb27c 100644 --- a/lnet/include/lnet/lib-types.h +++ b/lnet/include/lnet/lib-types.h @@ -509,10 +509,9 @@ struct lnet_ni { __u32 ni_sel_priority; /* - * equivalent interfaces to use - * This is an array because socklnd bonding can still be configured + * equivalent interface to use */ - char *ni_interfaces[LNET_INTERFACES_NUM]; + char *ni_interface; struct net *ni_net_ns; /* original net namespace */ }; diff --git a/lnet/include/uapi/linux/lnet/lnet-dlc.h b/lnet/include/uapi/linux/lnet/lnet-dlc.h index c37edc8..6481457 100644 --- a/lnet/include/uapi/linux/lnet/lnet-dlc.h +++ b/lnet/include/uapi/linux/lnet/lnet-dlc.h @@ -93,7 +93,7 @@ struct lnet_ioctl_config_lnd_tunables { }; struct lnet_ioctl_net_config { - char ni_interfaces[LNET_INTERFACES_NUM][LNET_MAX_STR_LEN]; + char ni_interface[LNET_MAX_STR_LEN]; __u32 ni_status; __u32 ni_cpts[LNET_MAX_SHOW_NUM_CPT]; char cfg_bulk[0]; @@ -222,12 +222,11 @@ struct lnet_ioctl_element_msg_stats { struct lnet_ioctl_config_ni { struct libcfs_ioctl_hdr lic_cfg_hdr; lnet_nid_t lic_nid; - char lic_ni_intf[LNET_INTERFACES_NUM][LNET_MAX_STR_LEN]; + char lic_ni_intf[LNET_MAX_STR_LEN]; char lic_legacy_ip2nets[LNET_MAX_STR_LEN]; __u32 lic_cpts[LNET_MAX_SHOW_NUM_CPT]; __u32 lic_ncpts; __u32 lic_status; - __u32 lic_tcp_bonding; __u32 lic_idx; __s32 lic_dev_cpt; char pad[4]; diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index 85d37b7..337b74e 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -3267,19 +3267,12 @@ kiblnd_startup(struct lnet_ni *ni) kiblnd_tunables_setup(ni); /* - * ni_interfaces is only to support legacy pre Multi-Rail - * tcp bonding for ksocklnd. Multi-Rail wants each secondary - * IP to be treated as an unique 'struct ni' interfaces instead. + * Multi-Rail wants each secondary + * IP to be treated as an unique 'struct ni' interface. */ - if (ni->ni_interfaces[0] != NULL) { + if (ni->ni_interface != NULL) { /* Use the IPoIB interface specified in 'networks=' */ - if (ni->ni_interfaces[1] != NULL) { - CERROR("ko2iblnd: Multiple interfaces not supported\n"); - rc = -EINVAL; - goto failed; - } - - ifname = ni->ni_interfaces[0]; + ifname = ni->ni_interface; } else { ifname = *kiblnd_tunables.kib_default_ipif; } diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index 4d3b7a4..5275147 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -48,16 +48,12 @@ static struct ksock_interface * ksocknal_ip2iface(struct lnet_ni *ni, struct sockaddr *addr) { struct ksock_net *net = ni->ni_data; - int i; struct ksock_interface *iface; - for (i = 0; i < net->ksnn_ninterfaces; i++) { - LASSERT(i < LNET_INTERFACES_NUM); - iface = &net->ksnn_interfaces[i]; + iface = &net->ksnn_interface; - if (rpc_cmp_addr((struct sockaddr *)&iface->ksni_addr, addr)) - return iface; - } + if (rpc_cmp_addr((struct sockaddr *)&iface->ksni_addr, addr)) + return iface; return NULL; } @@ -66,16 +62,12 @@ static struct ksock_interface * ksocknal_index2iface(struct lnet_ni *ni, int index) { struct ksock_net *net = ni->ni_data; - int i; struct ksock_interface *iface; - for (i = 0; i < net->ksnn_ninterfaces; i++) { - LASSERT(i < LNET_INTERFACES_NUM); - iface = &net->ksnn_interfaces[i]; + iface = &net->ksnn_interface; - if (iface->ksni_index == index) - return iface; - } + if (iface->ksni_index == index) + return iface; return NULL; } @@ -437,7 +429,6 @@ ksocknal_add_route_locked(struct ksock_peer_ni *peer_ni, struct ksock_route *rou LASSERT(!route->ksnr_scheduled); LASSERT(!route->ksnr_connecting); LASSERT(route->ksnr_connected == 0); - LASSERT(net->ksnn_ninterfaces > 0); /* LASSERT(unique) */ list_for_each(tmp, &peer_ni->ksnp_routes) { @@ -456,8 +447,8 @@ ksocknal_add_route_locked(struct ksock_peer_ni *peer_ni, struct ksock_route *rou ksocknal_peer_addref(peer_ni); /* set the route's interface to the current net's interface */ - route->ksnr_myiface = net->ksnn_interfaces[0].ksni_index; - net->ksnn_interfaces[0].ksni_nroutes++; + route->ksnr_myiface = net->ksnn_interface.ksni_index; + net->ksnn_interface.ksni_nroutes++; /* peer_ni's routelist takes over my ref on 'route' */ list_add_tail(&route->ksnr_list, &peer_ni->ksnp_routes); @@ -739,333 +730,6 @@ ksocknal_choose_scheduler_locked(unsigned int cpt) return sched; } -static int -ksocknal_local_ipvec(struct lnet_ni *ni, __u32 *ipaddrs) -{ - struct ksock_net *net = ni->ni_data; - int i, j; - int nip; - - read_lock(&ksocknal_data.ksnd_global_lock); - - nip = net->ksnn_ninterfaces; - LASSERT(nip <= LNET_INTERFACES_NUM); - - for (i = 0, j = 0; i < nip; i++) - if (net->ksnn_interfaces[i].ksni_addr.ss_family == AF_INET) { - struct sockaddr_in *sa = - (void *)&net->ksnn_interfaces[i].ksni_addr; - - ipaddrs[j] = ntohl(sa->sin_addr.s_addr); - LASSERT(ipaddrs[j] != 0); - j += 1; - } - nip = j; - - read_unlock(&ksocknal_data.ksnd_global_lock); - /* - * Only offer interfaces for additional connections if I have - * more than one. - */ - return nip < 2 ? 0 : nip; -} - -static int -ksocknal_match_peerip(struct ksock_interface *iface, __u32 *ips, int nips) -{ - int best_netmatch = 0; - int best_xor = 0; - int best = -1; - int this_xor; - int this_netmatch; - int i; - struct sockaddr_in *sa; - __u32 ip; - - sa = (struct sockaddr_in *)&iface->ksni_addr; - LASSERT(sa->sin_family == AF_INET); - ip = ntohl(sa->sin_addr.s_addr); - - for (i = 0; i < nips; i++) { - if (ips[i] == 0) - continue; - - this_xor = ips[i] ^ ip; - this_netmatch = ((this_xor & iface->ksni_netmask) == 0) ? 1 : 0; - - if (!(best < 0 || - best_netmatch < this_netmatch || - (best_netmatch == this_netmatch && - best_xor > this_xor))) - continue; - - best = i; - best_netmatch = this_netmatch; - best_xor = this_xor; - } - - LASSERT(best >= 0); - return best; -} - -static int -ksocknal_select_ips(struct ksock_peer_ni *peer_ni, __u32 *peerips, int n_peerips) -{ - rwlock_t *global_lock = &ksocknal_data.ksnd_global_lock; - struct ksock_net *net = peer_ni->ksnp_ni->ni_data; - struct ksock_interface *iface; - struct ksock_interface *best_iface; - int n_ips; - int i; - int j; - int k; - u32 ip; - u32 xor; - int this_netmatch; - int best_netmatch; - int best_npeers; - - /* CAVEAT EMPTOR: We do all our interface matching with an - * exclusive hold of global lock at IRQ priority. We're only - * expecting to be dealing with small numbers of interfaces, so the - * O(n**3)-ness shouldn't matter */ - - /* Also note that I'm not going to return more than n_peerips - * interfaces, even if I have more myself */ - - write_lock_bh(global_lock); - - LASSERT(n_peerips <= LNET_INTERFACES_NUM); - LASSERT(net->ksnn_ninterfaces <= LNET_INTERFACES_NUM); - - /* Only match interfaces for additional connections - * if I have > 1 interface - */ - n_ips = (net->ksnn_ninterfaces < 2) ? 0 : - min(n_peerips, net->ksnn_ninterfaces); - - for (i = 0; peer_ni->ksnp_n_passive_ips < n_ips; i++) { - /* ^ yes really... */ - - /* If we have any new interfaces, first tick off all the - * peer_ni IPs that match old interfaces, then choose new - * interfaces to match the remaining peer_ni IPS. - * We don't forget interfaces we've stopped using; we might - * start using them again... */ - - if (i < peer_ni->ksnp_n_passive_ips) { - /* Old interface. */ - struct sockaddr_in sa = { .sin_family = AF_INET}; - - sa.sin_addr.s_addr = - htonl(peer_ni->ksnp_passive_ips[i]); - best_iface = ksocknal_ip2iface(peer_ni->ksnp_ni, - (struct sockaddr *)&sa); - - /* peer_ni passive ips are kept up to date */ - LASSERT(best_iface != NULL); - } else { - /* choose a new interface */ - struct sockaddr_in *sa; - - LASSERT (i == peer_ni->ksnp_n_passive_ips); - - best_iface = NULL; - best_netmatch = 0; - best_npeers = 0; - - for (j = 0; j < net->ksnn_ninterfaces; j++) { - iface = &net->ksnn_interfaces[j]; - sa = (void *)&iface->ksni_addr; - if (sa->sin_family != AF_INET) - continue; - ip = ntohl(sa->sin_addr.s_addr); - - for (k = 0; k < peer_ni->ksnp_n_passive_ips; k++) - if (peer_ni->ksnp_passive_ips[k] == ip) - break; - - if (k < peer_ni->ksnp_n_passive_ips) /* using it already */ - continue; - - k = ksocknal_match_peerip(iface, peerips, n_peerips); - xor = (ip ^ peerips[k]); - this_netmatch = ((xor & iface->ksni_netmask) == 0) ? 1 : 0; - - if (!(best_iface == NULL || - best_netmatch < this_netmatch || - (best_netmatch == this_netmatch && - best_npeers > iface->ksni_npeers))) - continue; - - best_iface = iface; - best_netmatch = this_netmatch; - best_npeers = iface->ksni_npeers; - } - - LASSERT(best_iface != NULL); - - best_iface->ksni_npeers++; - sa = (void *)&best_iface->ksni_addr; - ip = ntohl(sa->sin_addr.s_addr); - peer_ni->ksnp_passive_ips[i] = ip; - peer_ni->ksnp_n_passive_ips = i+1; - } - - /* mark the best matching peer_ni IP used */ - j = ksocknal_match_peerip(best_iface, peerips, n_peerips); - peerips[j] = 0; - } - - /* Overwrite input peer_ni IP addresses */ - memcpy(peerips, peer_ni->ksnp_passive_ips, n_ips * sizeof(*peerips)); - - write_unlock_bh(global_lock); - - return (n_ips); -} - -static void -ksocknal_create_routes(struct ksock_peer_ni *peer_ni, int port, - __u32 *peer_ipaddrs, int npeer_ipaddrs) -{ - struct ksock_route *newroute = NULL; - rwlock_t *global_lock = &ksocknal_data.ksnd_global_lock; - struct lnet_ni *ni = peer_ni->ksnp_ni; - struct ksock_net *net = ni->ni_data; - struct list_head *rtmp; - struct ksock_route *route; - struct ksock_interface *iface; - struct ksock_interface *best_iface; - int best_netmatch; - int this_netmatch; - int best_nroutes; - int i; - int j; - - /* CAVEAT EMPTOR: We do all our interface matching with an - * exclusive hold of global lock at IRQ priority. We're only - * expecting to be dealing with small numbers of interfaces, so the - * O(n**3)-ness here shouldn't matter */ - - write_lock_bh(global_lock); - - if (net->ksnn_ninterfaces < 2) { - /* Only create additional connections - * if I have > 1 interface */ - write_unlock_bh(global_lock); - return; - } - - LASSERT(npeer_ipaddrs <= LNET_INTERFACES_NUM); - - for (i = 0; i < npeer_ipaddrs; i++) { - if (newroute) { - struct sockaddr_in *sa = (void *)&newroute->ksnr_addr; - - memset(sa, 0, sizeof(*sa)); - sa->sin_family = AF_INET; - sa->sin_addr.s_addr = htonl(peer_ipaddrs[i]); - } else { - struct sockaddr_in sa = {.sin_family = AF_INET}; - - write_unlock_bh(global_lock); - - sa.sin_addr.s_addr = htonl(peer_ipaddrs[i]); - sa.sin_port = htons(port); - newroute = - ksocknal_create_route((struct sockaddr *)&sa); - if (!newroute) - return; - - write_lock_bh(global_lock); - } - - if (peer_ni->ksnp_closing) { - /* peer_ni got closed under me */ - break; - } - - /* Already got a route? */ - route = NULL; - list_for_each(rtmp, &peer_ni->ksnp_routes) { - route = list_entry(rtmp, struct ksock_route, ksnr_list); - - if (rpc_cmp_addr( - (struct sockaddr *)&route->ksnr_addr, - (struct sockaddr *)&newroute->ksnr_addr)) - break; - - route = NULL; - } - if (route != NULL) - continue; - - best_iface = NULL; - best_nroutes = 0; - best_netmatch = 0; - - LASSERT(net->ksnn_ninterfaces <= LNET_INTERFACES_NUM); - - /* Select interface to connect from */ - for (j = 0; j < net->ksnn_ninterfaces; j++) { - __u32 iface_ip, route_ip; - - iface = &net->ksnn_interfaces[j]; - - /* Using this interface already? */ - list_for_each(rtmp, &peer_ni->ksnp_routes) { - route = list_entry(rtmp, struct ksock_route, - ksnr_list); - - if (route->ksnr_myiface == iface->ksni_index) - break; - - route = NULL; - } - if (route != NULL) - continue; - if (iface->ksni_addr.ss_family != AF_INET) - continue; - if (newroute->ksnr_addr.ss_family != AF_INET) - continue; - - iface_ip = - ntohl(((struct sockaddr_in *) - &iface->ksni_addr)->sin_addr.s_addr); - route_ip = - ntohl(((struct sockaddr_in *) - &newroute->ksnr_addr)->sin_addr.s_addr); - - this_netmatch = (((iface_ip ^ route_ip) & - iface->ksni_netmask) == 0) ? 1 : 0; - - if (!(best_iface == NULL || - best_netmatch < this_netmatch || - (best_netmatch == this_netmatch && - best_nroutes > iface->ksni_nroutes))) - continue; - - best_iface = iface; - best_netmatch = this_netmatch; - best_nroutes = iface->ksni_nroutes; - } - - if (best_iface == NULL) - continue; - - newroute->ksnr_myiface = best_iface->ksni_index; - best_iface->ksni_nroutes++; - - ksocknal_add_route_locked(peer_ni, newroute); - newroute = NULL; - } - - write_unlock_bh(global_lock); - if (newroute != NULL) - ksocknal_route_decref(newroute); -} - int ksocknal_accept(struct lnet_ni *ni, struct socket *sock) { @@ -1178,13 +842,13 @@ ksocknal_create_conn(struct lnet_ni *ni, struct ksock_route *route, * Passive connections use the listener timeout since the peer_ni sends * eagerly */ - if (active) { - peer_ni = route->ksnr_peer; - LASSERT(ni == peer_ni->ksnp_ni); + if (active) { + peer_ni = route->ksnr_peer; + LASSERT(ni == peer_ni->ksnp_ni); - /* Active connection sends HELLO eagerly */ - hello->kshm_nips = ksocknal_local_ipvec(ni, hello->kshm_ips); - peerid = peer_ni->ksnp_id; + /* Active connection sends HELLO eagerly */ + hello->kshm_nips = 0; + peerid = peer_ni->ksnp_id; write_lock_bh(global_lock); conn->ksnc_proto = peer_ni->ksnp_proto; @@ -1209,15 +873,15 @@ ksocknal_create_conn(struct lnet_ni *ni, struct ksock_route *route, /* Passive, get protocol from peer_ni */ conn->ksnc_proto = NULL; - } + } - rc = ksocknal_recv_hello (ni, conn, hello, &peerid, &incarnation); - if (rc < 0) - goto failed_1; + rc = ksocknal_recv_hello(ni, conn, hello, &peerid, &incarnation); + if (rc < 0) + goto failed_1; - LASSERT (rc == 0 || active); - LASSERT (conn->ksnc_proto != NULL); - LASSERT (peerid.nid != LNET_NID_ANY); + LASSERT(rc == 0 || active); + LASSERT(conn->ksnc_proto != NULL); + LASSERT(peerid.nid != LNET_NID_ANY); cpt = lnet_cpt_of_nid(peerid.nid, ni); @@ -1420,15 +1084,8 @@ ksocknal_create_conn(struct lnet_ni *ni, struct ksock_route *route, &conn->ksnc_myaddr, &conn->ksnc_peeraddr, incarnation, cpt); - if (active) { - /* additional routes after interface exchange? */ - ksocknal_create_routes( - peer_ni, - rpc_get_port((struct sockaddr *)&conn->ksnc_peeraddr), - hello->kshm_ips, hello->kshm_nips); - } else { - hello->kshm_nips = ksocknal_select_ips(peer_ni, hello->kshm_ips, - hello->kshm_nips); + if (!active) { + hello->kshm_nips = 0; rc = ksocknal_send_hello(ni, conn, peerid.nid, hello); } @@ -1996,181 +1653,26 @@ ksocknal_push(struct lnet_ni *ni, struct lnet_process_id id) return rc; } -static int -ksocknal_add_interface(struct lnet_ni *ni, __u32 ipaddress, __u32 netmask) -{ - struct ksock_net *net = ni->ni_data; - struct ksock_interface *iface; - struct sockaddr_in sa = { .sin_family = AF_INET }; - int rc; - int i; - int j; - struct ksock_peer_ni *peer_ni; - struct list_head *rtmp; - struct ksock_route *route; - - if (ipaddress == 0 || - netmask == 0) - return -EINVAL; - - write_lock_bh(&ksocknal_data.ksnd_global_lock); - - sa.sin_addr.s_addr = htonl(ipaddress); - iface = ksocknal_ip2iface(ni, (struct sockaddr *)&sa); - if (iface != NULL) { - /* silently ignore dups */ - rc = 0; - } else if (net->ksnn_ninterfaces == LNET_INTERFACES_NUM) { - rc = -ENOSPC; - } else { - iface = &net->ksnn_interfaces[net->ksnn_ninterfaces++]; - - iface->ksni_index = ksocknal_ip2index((struct sockaddr *)&sa, - ni); - rpc_copy_addr((struct sockaddr *)&iface->ksni_addr, - (struct sockaddr *)&sa); - iface->ksni_netmask = netmask; - iface->ksni_nroutes = 0; - iface->ksni_npeers = 0; - - hash_for_each(ksocknal_data.ksnd_peers, i, peer_ni, ksnp_list) { - for (j = 0; j < peer_ni->ksnp_n_passive_ips; j++) - if (peer_ni->ksnp_passive_ips[j] == ipaddress) - iface->ksni_npeers++; - - list_for_each(rtmp, &peer_ni->ksnp_routes) { - route = list_entry(rtmp, - struct ksock_route, - ksnr_list); - - if (route->ksnr_myiface == - iface->ksni_index) - iface->ksni_nroutes++; - } - } - - rc = 0; - /* NB only new connections will pay attention to the new - * interface! - */ - } - - write_unlock_bh(&ksocknal_data.ksnd_global_lock); - - return rc; -} - -static void -ksocknal_peer_del_interface_locked(struct ksock_peer_ni *peer_ni, - __u32 ipaddr, int index) -{ - struct ksock_route *route; - struct ksock_route *rnxt; - struct ksock_conn *conn; - struct ksock_conn *cnxt; - int i; - int j; - - for (i = 0; i < peer_ni->ksnp_n_passive_ips; i++) - if (peer_ni->ksnp_passive_ips[i] == ipaddr) { - for (j = i+1; j < peer_ni->ksnp_n_passive_ips; j++) - peer_ni->ksnp_passive_ips[j-1] = - peer_ni->ksnp_passive_ips[j]; - peer_ni->ksnp_n_passive_ips--; - break; - } - - list_for_each_entry_safe(route, rnxt, &peer_ni->ksnp_routes, - ksnr_list) { - if (route->ksnr_myiface != index) - continue; - - if (route->ksnr_share_count != 0) { - /* Manually created; keep, but unbind */ - route->ksnr_myiface = -1; - } else { - ksocknal_del_route_locked(route); - } - } - - list_for_each_entry_safe(conn, cnxt, &peer_ni->ksnp_conns, ksnc_list) - if (conn->ksnc_route->ksnr_myiface == index) - ksocknal_close_conn_locked (conn, 0); -} - -static int -ksocknal_del_interface(struct lnet_ni *ni, __u32 ipaddress) -{ - struct ksock_net *net = ni->ni_data; - int rc = -ENOENT; - struct hlist_node *nxt; - struct ksock_peer_ni *peer_ni; - u32 this_ip; - struct sockaddr_in sa = {.sin_family = AF_INET }; - int index; - int i; - int j; - - sa.sin_addr.s_addr = htonl(ipaddress); - index = ksocknal_ip2index((struct sockaddr *)&sa, ni); - - write_lock_bh(&ksocknal_data.ksnd_global_lock); - - for (i = 0; i < net->ksnn_ninterfaces; i++) { - struct sockaddr_in *sa = - (void *)&net->ksnn_interfaces[i].ksni_addr; - - if (sa->sin_family != AF_INET) - continue; - this_ip = ntohl(sa->sin_addr.s_addr); - - if (!(ipaddress == 0 || - ipaddress == this_ip)) - continue; - - rc = 0; - - for (j = i+1; j < net->ksnn_ninterfaces; j++) - net->ksnn_interfaces[j-1] = - net->ksnn_interfaces[j]; - - net->ksnn_ninterfaces--; - - hash_for_each_safe(ksocknal_data.ksnd_peers, j, - nxt, peer_ni, ksnp_list) { - if (peer_ni->ksnp_ni != ni) - continue; - - ksocknal_peer_del_interface_locked(peer_ni, - this_ip, index); - } - } - - write_unlock_bh(&ksocknal_data.ksnd_global_lock); - - return rc; -} - int ksocknal_ctl(struct lnet_ni *ni, unsigned int cmd, void *arg) { struct lnet_process_id id = {0}; - struct libcfs_ioctl_data *data = arg; - int rc; + struct libcfs_ioctl_data *data = arg; + int rc; - switch(cmd) { - case IOC_LIBCFS_GET_INTERFACE: { + switch(cmd) { + case IOC_LIBCFS_GET_INTERFACE: { struct ksock_net *net = ni->ni_data; struct ksock_interface *iface; struct sockaddr_in *sa; read_lock(&ksocknal_data.ksnd_global_lock); - if (data->ioc_count >= (__u32)net->ksnn_ninterfaces) { + if (data->ioc_count >= 1) { rc = -ENOENT; } else { rc = 0; - iface = &net->ksnn_interfaces[data->ioc_count]; + iface = &net->ksnn_interface; sa = (void *)&iface->ksni_addr; if (sa->sin_family == AF_INET) @@ -2183,18 +1685,9 @@ ksocknal_ctl(struct lnet_ni *ni, unsigned int cmd, void *arg) } read_unlock(&ksocknal_data.ksnd_global_lock); - return rc; + return rc; } - case IOC_LIBCFS_ADD_INTERFACE: - return ksocknal_add_interface(ni, - data->ioc_u32[0], /* IP address */ - data->ioc_u32[1]); /* net mask */ - - case IOC_LIBCFS_DEL_INTERFACE: - return ksocknal_del_interface(ni, - data->ioc_u32[0]); /* IP address */ - case IOC_LIBCFS_GET_PEER: { __u32 myip = 0; __u32 ip = 0; @@ -2561,7 +2054,6 @@ ksocknal_shutdown(struct lnet_ni *ni) .nid = LNET_NID_ANY, .pid = LNET_PID_ANY, }; - int i; LASSERT(ksocknal_data.ksnd_init == SOCKNAL_INIT_ALL); LASSERT(ksocknal_data.ksnd_nnets > 0); @@ -2581,10 +2073,8 @@ ksocknal_shutdown(struct lnet_ni *ni) atomic_read(&net->ksnn_npeers) - SOCKNAL_SHUTDOWN_BIAS); - for (i = 0; i < net->ksnn_ninterfaces; i++) { - LASSERT(net->ksnn_interfaces[i].ksni_npeers == 0); - LASSERT(net->ksnn_interfaces[i].ksni_nroutes == 0); - } + LASSERT(net->ksnn_interface.ksni_npeers == 0); + LASSERT(net->ksnn_interface.ksni_nroutes == 0); list_del(&net->ksnn_list); LIBCFS_FREE(net, sizeof(*net)); @@ -2598,41 +2088,30 @@ static int ksocknal_search_new_ipif(struct ksock_net *net) { int new_ipif = 0; - int i; + char *ifnam = &net->ksnn_interface.ksni_name[0]; + char *colon = strchr(ifnam, ':'); + bool found = false; + struct ksock_net *tmp; - for (i = 0; i < net->ksnn_ninterfaces; i++) { - char *ifnam = &net->ksnn_interfaces[i].ksni_name[0]; - char *colon = strchr(ifnam, ':'); - bool found = false; - struct ksock_net *tmp; - int j; - - if (colon != NULL) /* ignore alias device */ - *colon = 0; - - list_for_each_entry(tmp, &ksocknal_data.ksnd_nets, - ksnn_list) { - for (j = 0; !found && j < tmp->ksnn_ninterfaces; j++) { - char *ifnam2 = &tmp->ksnn_interfaces[j].\ - ksni_name[0]; - char *colon2 = strchr(ifnam2, ':'); - - if (colon2 != NULL) - *colon2 = 0; - - found = strcmp(ifnam, ifnam2) == 0; - if (colon2 != NULL) - *colon2 = ':'; - } - if (found) - break; - } + if (colon != NULL) + *colon = 0; - new_ipif += !found; - if (colon != NULL) - *colon = ':'; + list_for_each_entry(tmp, &ksocknal_data.ksnd_nets, ksnn_list) { + char *ifnam2 = &tmp->ksnn_interface.ksni_name[0]; + char *colon2 = strchr(ifnam2, ':'); + + if (colon2 != NULL) + *colon2 = 0; + + found = strcmp(ifnam, ifnam2) == 0; + if (colon2 != NULL) + *colon2 = ':'; } + new_ipif += !found; + if (colon != NULL) + *colon = ':'; + return new_ipif; } @@ -2715,25 +2194,22 @@ ksocknal_startup(struct lnet_ni *ni) struct lnet_ioctl_config_lnd_cmn_tunables *net_tunables; struct ksock_interface *ksi = NULL; struct lnet_inetdev *ifaces = NULL; + struct sockaddr_in *sa; int i = 0; int rc; LASSERT (ni->ni_net->net_lnd == &the_ksocklnd); - if (ksocknal_data.ksnd_init == SOCKNAL_INIT_NOTHING) { rc = ksocknal_base_startup(); if (rc != 0) return rc; } - LIBCFS_ALLOC(net, sizeof(*net)); if (net == NULL) goto fail_0; - net->ksnn_incarnation = ktime_get_real_ns(); ni->ni_data = net; net_tunables = &ni->ni_net->net_tunables; - if (net_tunables->lct_peer_timeout == -1) net_tunables->lct_peer_timeout = *ksocknal_tunables.ksnd_peertimeout; @@ -2759,82 +2235,31 @@ ksocknal_startup(struct lnet_ni *ni) if (rc < 0) goto fail_1; - if (!ni->ni_interfaces[0]) { - struct sockaddr_in *sa; - - ksi = &net->ksnn_interfaces[0]; - sa = (void *)&ksi->ksni_addr; - - /* Use the first discovered interface */ - net->ksnn_ninterfaces = 1; - ni->ni_dev_cpt = ifaces[0].li_cpt; - memset(sa, 0, sizeof(*sa)); - sa->sin_family = AF_INET; - sa->sin_addr.s_addr = htonl(ifaces[0].li_ipaddr); - ksi->ksni_index = ksocknal_ip2index((struct sockaddr *)sa, ni); - ksi->ksni_netmask = ifaces[0].li_netmask; - strlcpy(ksi->ksni_name, ifaces[0].li_name, - sizeof(ksi->ksni_name)); - } else { - /* Before Multi-Rail ksocklnd would manage - * multiple interfaces with its own tcp bonding. - * If we encounter an old configuration using - * this tcp bonding approach then we need to - * handle more than one ni_interfaces. - * - * In Multi-Rail configuration only ONE ni_interface - * should exist. Each IP alias should be mapped to - * each 'struct net_ni'. - */ - for (i = 0; i < LNET_INTERFACES_NUM; i++) { - int j; + ksi = &net->ksnn_interface; - if (!ni->ni_interfaces[i]) + /* Use the first discovered interface or look in the list */ + if (ni->ni_interface) { + for (i = 0; i < rc; i++) + if (strcmp(ifaces[i].li_name, ni->ni_interface) == 0) break; - for (j = 0; j < LNET_INTERFACES_NUM; j++) { - if (i != j && ni->ni_interfaces[j] && - strcmp(ni->ni_interfaces[i], - ni->ni_interfaces[j]) == 0) { - rc = -EEXIST; - CERROR("ksocklnd: found duplicate %s at %d and %d, rc = %d\n", - ni->ni_interfaces[i], i, j, rc); - goto fail_1; - } - } - - for (j = 0; j < rc; j++) { - struct sockaddr_in *sa; - - if (strcmp(ifaces[j].li_name, - ni->ni_interfaces[i]) != 0) - continue; - - ksi = - &net->ksnn_interfaces[net->ksnn_ninterfaces]; - sa = (void *)&ksi->ksni_addr; - ni->ni_dev_cpt = ifaces[j].li_cpt; - memset(sa, 0, sizeof(*sa)); - sa->sin_family = AF_INET; - sa->sin_addr.s_addr = - htonl(ifaces[j].li_ipaddr); - ksi->ksni_index = ksocknal_ip2index( - (struct sockaddr *)sa, ni); - ksi->ksni_netmask = ifaces[j].li_netmask; - strlcpy(ksi->ksni_name, ifaces[j].li_name, - sizeof(ksi->ksni_name)); - net->ksnn_ninterfaces++; - break; - } - } - /* ni_interfaces don't map to all network interfaces */ - if (!ksi || net->ksnn_ninterfaces != i) { - CERROR("ksocklnd: requested %d but only %d interfaces found\n", - i, net->ksnn_ninterfaces); + /* ni_interfaces doesn't contain the interface we want */ + if (i == rc) { + CERROR("ksocklnd: failed to find interface %s\n", + ni->ni_interface); goto fail_1; } } + ni->ni_dev_cpt = ifaces[i].li_cpt; + sa = (void *)&ksi->ksni_addr; + memset(sa, 0, sizeof(*sa)); + sa->sin_family = AF_INET; + sa->sin_addr.s_addr = htonl(ifaces[i].li_ipaddr); + ksi->ksni_index = ksocknal_ip2index((struct sockaddr *)sa, ni); + ksi->ksni_netmask = ifaces[i].li_netmask; + strlcpy(ksi->ksni_name, ifaces[i].li_name, sizeof(ksi->ksni_name)); + /* call it before add it to ksocknal_data.ksnd_nets */ rc = ksocknal_net_start_threads(net, ni->ni_cpts, ni->ni_ncpts); if (rc != 0) @@ -2847,7 +2272,6 @@ ksocknal_startup(struct lnet_ni *ni) ntohl(((struct sockaddr_in *) &ksi->ksni_addr)->sin_addr.s_addr)); list_add(&net->ksnn_list, &ksocknal_data.ksnd_nets); - ksocknal_data.ksnd_nnets++; return 0; diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h index 9e9aafa..cd95ea6 100644 --- a/lnet/klnds/socklnd/socklnd.h +++ b/lnet/klnds/socklnd/socklnd.h @@ -162,8 +162,7 @@ struct ksock_net { __u64 ksnn_incarnation; /* my epoch */ struct list_head ksnn_list; /* chain on global list */ atomic_t ksnn_npeers; /* # peers */ - int ksnn_ninterfaces; /* IP interfaces */ - struct ksock_interface ksnn_interfaces[LNET_INTERFACES_NUM]; + struct ksock_interface ksnn_interface; /* IP interface */ }; /* When the ksock_net is shut down, this (negative) bias is added to * ksnn_npeers, which prevents new peers from being added. diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index 1b7cb7e..322565a 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -72,10 +72,10 @@ static int rnet_htable_size = LNET_REMOTE_NETS_HASH_DEFAULT; module_param(rnet_htable_size, int, 0444); MODULE_PARM_DESC(rnet_htable_size, "size of remote network hash table"); -static int use_tcp_bonding = false; +static int use_tcp_bonding; module_param(use_tcp_bonding, int, 0444); MODULE_PARM_DESC(use_tcp_bonding, - "use_tcp_bonding parameter has been deprecated"); + "use_tcp_bonding parameter has been removed"); unsigned int lnet_numa_range = 0; module_param(lnet_numa_range, uint, 0444); @@ -2557,19 +2557,6 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun) * After than we want to delete the network being added, * to avoid a memory leak. */ - - /* - * When a network uses TCP bonding then all its interfaces - * must be specified when the network is first defined: the - * TCP bonding code doesn't allow for interfaces to be added - * or removed. - */ - if (net_l != net && net_l != NULL && use_tcp_bonding && - LNET_NETTYP(net_l->net_id) == SOCKLND) { - rc = -EINVAL; - goto failed0; - } - while (!list_empty(&net->net_ni_added)) { ni = list_entry(net->net_ni_added.next, struct lnet_ni, ni_netlist); @@ -2578,7 +2565,7 @@ lnet_startup_lndnet(struct lnet_net *net, struct lnet_lnd_tunables *tun) /* make sure that the the NI we're about to start * up is actually unique. if it's not fail. */ if (!lnet_ni_unique_net(&net_l->net_ni_list, - ni->ni_interfaces[0])) { + ni->ni_interface)) { rc = -EEXIST; goto failed1; } @@ -2822,7 +2809,7 @@ LNetNIInit(lnet_pid_t requested_pid) } if (use_tcp_bonding) - CWARN("'use_tcp_bonding' option has been deprecated. See LU-13641\n"); + CWARN("use_tcp_bonding has been removed. Use Multi-Rail and Dynamic Discovery instead, see LU-13641\n"); /* If LNet is being initialized via DLC it is possible * that the user requests not to load module parameters (ones which @@ -2831,8 +2818,7 @@ LNetNIInit(lnet_pid_t requested_pid) * in this case. On cleanup in case of failure only clean up * routes if it has been loaded */ if (!the_lnet.ln_nis_from_mod_params) { - rc = lnet_parse_networks(&net_head, lnet_get_networks(), - use_tcp_bonding); + rc = lnet_parse_networks(&net_head, lnet_get_networks()); if (rc < 0) goto err_empty_list; } @@ -2984,14 +2970,10 @@ lnet_fill_ni_info(struct lnet_ni *ni, struct lnet_ioctl_config_ni *cfg_ni, if (!ni || !cfg_ni || !tun) return; - if (ni->ni_interfaces[0] != NULL) { - for (i = 0; i < ARRAY_SIZE(ni->ni_interfaces); i++) { - if (ni->ni_interfaces[i] != NULL) { - strncpy(cfg_ni->lic_ni_intf[i], - ni->ni_interfaces[i], - sizeof(cfg_ni->lic_ni_intf[i])); - } - } + if (ni->ni_interface != NULL) { + strncpy(cfg_ni->lic_ni_intf, + ni->ni_interface, + sizeof(cfg_ni->lic_ni_intf)); } cfg_ni->lic_nid = ni->ni_nid; @@ -2999,7 +2981,6 @@ lnet_fill_ni_info(struct lnet_ni *ni, struct lnet_ioctl_config_ni *cfg_ni, cfg_ni->lic_status = LNET_NI_STATUS_UP; else cfg_ni->lic_status = ni->ni_status->ns_status; - cfg_ni->lic_tcp_bonding = use_tcp_bonding; cfg_ni->lic_dev_cpt = ni->ni_dev_cpt; memcpy(&tun->lt_cmn, &ni->ni_net->net_tunables, sizeof(tun->lt_cmn)); @@ -3065,17 +3046,12 @@ lnet_fill_ni_info_legacy(struct lnet_ni *ni, if (!net_config) return; - BUILD_BUG_ON(ARRAY_SIZE(ni->ni_interfaces) != - ARRAY_SIZE(net_config->ni_interfaces)); - - for (i = 0; i < ARRAY_SIZE(ni->ni_interfaces); i++) { - if (!ni->ni_interfaces[i]) - break; + if (!ni->ni_interface) + return; - strncpy(net_config->ni_interfaces[i], - ni->ni_interfaces[i], - sizeof(net_config->ni_interfaces[i])); - } + strncpy(net_config->ni_interface, + ni->ni_interface, + sizeof(net_config->ni_interface)); config->cfg_nid = ni->ni_nid; config->cfg_config_u.cfg_net.net_peer_timeout = @@ -3418,7 +3394,7 @@ static int lnet_handle_legacy_ip2nets(char *ip2nets, if (rc < 0) return rc; - rc = lnet_parse_networks(&net_head, nets, use_tcp_bonding); + rc = lnet_parse_networks(&net_head, nets); if (rc < 0) return rc; @@ -3481,7 +3457,7 @@ int lnet_dyn_add_ni(struct lnet_ioctl_config_ni *conf) } ni = lnet_ni_alloc_w_cpt_array(net, conf->lic_cpts, conf->lic_ncpts, - conf->lic_ni_intf[0]); + conf->lic_ni_intf); if (!ni) return -ENOMEM; @@ -3600,7 +3576,7 @@ lnet_dyn_add_net(struct lnet_ioctl_config_data *conf) const char *nets = conf->cfg_config_u.cfg_net.net_intf; /* Create a net/ni structures for the network string */ - rc = lnet_parse_networks(&net_head, nets, use_tcp_bonding); + rc = lnet_parse_networks(&net_head, nets); if (rc <= 0) return rc == 0 ? -EINVAL : rc; diff --git a/lnet/lnet/config.c b/lnet/lnet/config.c index 610100a..1f93709 100644 --- a/lnet/lnet/config.c +++ b/lnet/lnet/config.c @@ -113,29 +113,13 @@ lnet_ni_unique_net(struct list_head *nilist, char *iface) list_for_each(tmp, nilist) { ni = list_entry(tmp, struct lnet_ni, ni_netlist); - if (ni->ni_interfaces[0] != NULL && - strncmp(ni->ni_interfaces[0], iface, strlen(iface)) == 0) + if (ni->ni_interface != NULL && + strncmp(ni->ni_interface, iface, strlen(iface)) == 0) return false; } return true; } - -/* check that the NI is unique to the interfaces with in the same NI. - * This is only a consideration if use_tcp_bonding is set */ -static bool -lnet_ni_unique_ni(char *iface_list[LNET_INTERFACES_NUM], char *iface) -{ - int i; - for (i = 0; i < LNET_INTERFACES_NUM; i++) { - if (iface_list[i] != NULL && - strncmp(iface_list[i], iface, strlen(iface)) == 0) - return false; - } - - return true; -} - static bool in_array(__u32 *array, __u32 size, __u32 value) { @@ -294,8 +278,6 @@ lnet_net_remove_cpts(__u32 *cpts, __u32 ncpts, struct lnet_net *net) void lnet_ni_free(struct lnet_ni *ni) { - int i; - lnet_net_remove_cpts(ni->ni_cpts, ni->ni_ncpts, ni->ni_net); if (ni->ni_refs != NULL) @@ -307,10 +289,9 @@ lnet_ni_free(struct lnet_ni *ni) if (ni->ni_cpts != NULL) cfs_expr_list_values_free(ni->ni_cpts, ni->ni_ncpts); - for (i = 0; i < LNET_INTERFACES_NUM && - ni->ni_interfaces[i] != NULL; i++) { - LIBCFS_FREE(ni->ni_interfaces[i], - strlen(ni->ni_interfaces[i]) + 1); + if (ni->ni_interface != NULL) { + LIBCFS_FREE(ni->ni_interface, + strlen(ni->ni_interface) + 1); } /* release reference to net namespace */ @@ -401,35 +382,31 @@ lnet_ni_add_interface(struct lnet_ni *ni, char *iface) if (ni == NULL) return -ENOMEM; - if (!lnet_ni_unique_ni(ni->ni_interfaces, iface)) - return -EINVAL; - /* Allocate a separate piece of memory and copy * into it the string, so we don't have * a depencency on the tokens string. This way we * can free the tokens at the end of the function. - * The newly allocated ni_interfaces[] can be + * The newly allocated ni_interface can be * freed when freeing the NI */ - while (niface < LNET_INTERFACES_NUM && - ni->ni_interfaces[niface] != NULL) + if (ni->ni_interface != NULL) niface++; - if (niface >= LNET_INTERFACES_NUM) { + if (niface >= 1) { LCONSOLE_ERROR_MSG(0x115, "Too many interfaces " "for net %s\n", libcfs_net2str(LNET_NIDNET(ni->ni_nid))); return -EINVAL; } - LIBCFS_ALLOC(ni->ni_interfaces[niface], + LIBCFS_ALLOC(ni->ni_interface, strlen(iface) + 1); - if (ni->ni_interfaces[niface] == NULL) { + if (ni->ni_interface == NULL) { CERROR("Can't allocate net interface name\n"); return -ENOMEM; } - strncpy(ni->ni_interfaces[niface], iface, + strncpy(ni->ni_interface, iface, strlen(iface) + 1); return 0; @@ -582,8 +559,7 @@ failed: * nilist. */ int -lnet_parse_networks(struct list_head *netlist, const char *networks, - bool use_tcp_bonding) +lnet_parse_networks(struct list_head *netlist, const char *networks) { struct cfs_expr_list *net_el = NULL; struct cfs_expr_list *ni_el = NULL; @@ -725,8 +701,7 @@ lnet_parse_networks(struct list_head *netlist, const char *networks, if (IS_ERR_OR_NULL(net)) goto failed; - if (!nistr || - (use_tcp_bonding && LNET_NETTYP(net_id) == SOCKLND)) { + if (!nistr) { /* * No interface list was specified, allocate a * ni using the defaults. @@ -805,16 +780,9 @@ lnet_parse_networks(struct list_head *netlist, const char *networks, goto failed_syntax; } - if (use_tcp_bonding && - LNET_NETTYP(net->net_id) == SOCKLND) { - rc = lnet_ni_add_interface(ni, name); - if (rc != 0) - goto failed; - } else { - ni = lnet_ni_alloc(net, ni_el, name); - if (IS_ERR_OR_NULL(ni)) - goto failed; - } + ni = lnet_ni_alloc(net, ni_el, name); + if (IS_ERR_OR_NULL(ni)) + goto failed; if (ni_el) { if (ni_el != net_el) { diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c index a4e446a..c815732 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.c +++ b/lnet/utils/lnetconfig/liblnetconfig.c @@ -1458,7 +1458,7 @@ lustre_lnet_ioctl_config_ni(struct list_head *intf_list, LIBCFS_IOC_INIT_V2(*conf, lic_cfg_hdr); conf->lic_cfg_hdr.ioc_len = len; conf->lic_nid = nids[i]; - strncpy(conf->lic_ni_intf[0], intf_descr->intf_name, + strncpy(conf->lic_ni_intf, intf_descr->intf_name, LNET_MAX_STR_LEN); if (intf_descr->cpt_expr != NULL) @@ -2118,22 +2118,15 @@ int lustre_lnet_show_net(char *nw, int detail, int seq_no, /* don't add interfaces unless there is at least one * interface */ - if (strlen(ni_data->lic_ni_intf[0]) > 0) { + if (strlen(ni_data->lic_ni_intf) > 0) { interfaces = cYAML_create_object(item, "interfaces"); if (interfaces == NULL) goto out; - for (j = 0; j < LNET_INTERFACES_NUM; j++) { - if (strlen(ni_data->lic_ni_intf[j]) > 0) { - snprintf(str_buf, - sizeof(str_buf), "%d", j); - if (cYAML_create_string(interfaces, - str_buf, - ni_data->lic_ni_intf[j]) == - NULL) - goto out; - } - } + snprintf(str_buf, sizeof(str_buf), "%d", 0); + if (cYAML_create_string(interfaces, str_buf, + ni_data->lic_ni_intf) == NULL) + goto out; } if (detail) { @@ -2285,12 +2278,6 @@ continue_without_msg_stats: ni_data->lic_dev_cpt) == NULL) goto out; - if (!backup && - cYAML_create_number(item, "tcp bonding", - ni_data->lic_tcp_bonding) - == NULL) - goto out; - /* out put the CPTs in the format: "[x,x,x,...]" */ pos = str_buf; limit = str_buf + str_buf_len - 3;