From 0b406c91d175b6cdbdf472b70a37e618cd66a72d Mon Sep 17 00:00:00 2001 From: James Simmons Date: Thu, 5 Jan 2023 12:59:24 -0500 Subject: [PATCH] LU-13642 lnet: modify lnet_inetdev to work with large NIDS Change li_ipv6 field in struct lnet_inetdev to li_size which now represents the size of the NID address. This will work with the GUID of Inifiniband as well. Second change is to store li_ipaddr always in network format. This will allow direct comparsion between li_ipaddr and the nid_addr of struct lnet_nid. We will ensure AF_IB will also be in the same format as what will be stored in struct lnet_nid. Implement setup with a NID address for the ko2iblnd LND driver. Test-Parameters: trivial testlist=sanity-lnet Change-Id: I7c27edb67263dd5bda4728c536aee266d38a4592 Signed-off-by: James Simmons Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/49525 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Serguei Smirnov Reviewed-by: Neil Brown Reviewed-by: Oleg Drokin --- lnet/include/lnet/lib-lnet.h | 2 +- lnet/include/uapi/linux/lnet/lnet-types.h | 10 ---------- lnet/klnds/o2iblnd/o2iblnd.c | 19 ++++++++++-------- lnet/klnds/socklnd/socklnd.c | 6 +++--- lnet/lnet/config.c | 32 +++++++++++++++---------------- 5 files changed, 31 insertions(+), 38 deletions(-) diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 1a16e43..68dfc5c 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -908,7 +908,7 @@ struct lnet_inetdev { }; u32 li_index; bool li_iff_master; - bool li_ipv6; + u32 li_size; char li_name[IFNAMSIZ]; }; diff --git a/lnet/include/uapi/linux/lnet/lnet-types.h b/lnet/include/uapi/linux/lnet/lnet-types.h index 4620e2f..c8a61ab 100644 --- a/lnet/include/uapi/linux/lnet/lnet-types.h +++ b/lnet/include/uapi/linux/lnet/lnet-types.h @@ -115,16 +115,6 @@ static inline bool nid_is_nid4(const struct lnet_nid *nid) return NID_ADDR_BYTES(nid) == 4; } -static inline bool nid_is_ipv4(const struct lnet_nid *nid) -{ - return NID_ADDR_BYTES(nid) == 4; -} - -static inline bool nid_is_ipv6(const struct lnet_nid *nid) -{ - return NID_ADDR_BYTES(nid) == 16; -} - /* check for address set */ static inline bool nid_addr_is_set(const struct lnet_nid *nid) { diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index 3f03a51..e1c82e5 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -3577,8 +3577,7 @@ kiblnd_startup(struct lnet_ni *ni) kiblnd_tunables_setup(ni); - /* - * Multi-Rail wants each secondary + /* Multi-Rail wants each secondary * IP to be treated as an unique 'struct ni' interface. */ if (ni->ni_interface != NULL) { @@ -3586,6 +3585,9 @@ kiblnd_startup(struct lnet_ni *ni) ifname = ni->ni_interface; } else { ifname = *kiblnd_tunables.kib_default_ipif; + rc = libcfs_strnid(&ni->ni_nid, ifname); + if (rc < 0 || ni->ni_nid.nid_type != O2IBLND) + memset(&ni->ni_nid, 0, sizeof(ni->ni_nid)); } if (strlen(ifname) >= sizeof(ibdev->ibd_ifname)) { @@ -3598,12 +3600,13 @@ kiblnd_startup(struct lnet_ni *ni) if (rc < 0) goto failed; - for (i = 0; i < rc; i++) { - if (strcmp(ifname, ifaces[i].li_name) == 0) - break; - } + i = lnet_inet_select(ni, ifaces, rc); + if (i < 0) + goto failed; - if (i == rc) { + if (nid_addr_is_set(&ni->ni_nid)) { + strscpy(ifname, ifaces[i].li_name, sizeof(ifname)); + } else if (strcmp(ifname, ifaces[i].li_name) != 0) { CERROR("ko2iblnd: No matching interfaces\n"); rc = -ENOENT; goto failed; @@ -3619,7 +3622,7 @@ kiblnd_startup(struct lnet_ni *ni) goto failed; } - ibdev->ibd_ifip = ifaces[i].li_ipaddr; + ibdev->ibd_ifip = ntohl(ifaces[i].li_ipaddr); strlcpy(ibdev->ibd_ifname, ifaces[i].li_name, sizeof(ibdev->ibd_ifname)); ibdev->ibd_can_failover = ifaces[i].li_iff_master; diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index 1d15078..215d3de 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -2570,7 +2570,7 @@ ksocknal_startup(struct lnet_ni *ni) ni->ni_dev_cpt = ifaces[if_idx].li_cpt; ksi->ksni_index = ifaces[if_idx].li_index; - if (ifaces[if_idx].li_ipv6) { + if (ifaces[if_idx].li_size == sizeof(struct in6_addr)) { struct sockaddr_in6 *sa; sa = (void *)&ksi->ksni_addr; memset(sa, 0, sizeof(*sa)); @@ -2585,9 +2585,9 @@ ksocknal_startup(struct lnet_ni *ni) sa = (void *)&ksi->ksni_addr; memset(sa, 0, sizeof(*sa)); sa->sin_family = AF_INET; - sa->sin_addr.s_addr = htonl(ifaces[if_idx].li_ipaddr); + sa->sin_addr.s_addr = ifaces[if_idx].li_ipaddr; ksi->ksni_netmask = ifaces[if_idx].li_netmask; - ni->ni_nid.nid_size = 4 - 4; + ni->ni_nid.nid_size = 0; ni->ni_nid.nid_addr[0] = sa->sin_addr.s_addr; } strlcpy(ksi->ksni_name, ifaces[if_idx].li_name, sizeof(ksi->ksni_name)); diff --git a/lnet/lnet/config.c b/lnet/lnet/config.c index 94ac211..fdc1aa4b 100644 --- a/lnet/lnet/config.c +++ b/lnet/lnet/config.c @@ -1563,9 +1563,9 @@ int lnet_inet_enumerate(struct lnet_inetdev **dev_list, struct net *ns, bool v6) ifaces[nip].li_cpt = cpt; ifaces[nip].li_iff_master = !!(flags & IFF_MASTER); - ifaces[nip].li_ipv6 = false; + ifaces[nip].li_size = sizeof(ifa->ifa_local); ifaces[nip].li_index = dev->ifindex; - ifaces[nip].li_ipaddr = ntohl(ifa->ifa_local); + ifaces[nip].li_ipaddr = ifa->ifa_local; ifaces[nip].li_netmask = ntohl(ifa->ifa_mask); strlcpy(ifaces[nip].li_name, ifa->ifa_label, sizeof(ifaces[nip].li_name)); @@ -1605,7 +1605,7 @@ int lnet_inet_enumerate(struct lnet_inetdev **dev_list, struct net *ns, bool v6) ifaces[nip].li_cpt = cpt; ifaces[nip].li_iff_master = !!(flags & IFF_MASTER); - ifaces[nip].li_ipv6 = true; + ifaces[nip].li_size = sizeof(struct in6_addr); ifaces[nip].li_index = dev->ifindex; memcpy(ifaces[nip].li_ipv6addr, &ifa6->addr, sizeof(struct in6_addr)); @@ -1655,16 +1655,14 @@ int lnet_inet_select(struct lnet_ni *ni, /* IP unspecified, use IP of first matching interface */ break; - if (ifaces[if_idx].li_ipv6 && - nid_is_ipv6(&ni->ni_nid)) { - if (memcmp(ni->ni_nid.nid_addr, - ifaces[if_idx].li_ipv6addr, - sizeof(struct in6_addr)) == 0) - break; - } else if (!ifaces[if_idx].li_ipv6 && - nid_is_ipv4(&ni->ni_nid)) { - if (ni->ni_nid.nid_addr[0] == - htonl(ifaces[if_idx].li_ipaddr)) + if (ifaces[if_idx].li_size == NID_ADDR_BYTES(&ni->ni_nid)) { + char *addr = (char *)&ifaces[if_idx].li_ipaddr; + + if (ifaces[if_idx].li_size != 4) + addr = (char *)ifaces[if_idx].li_ipv6addr; + + if (memcmp(ni->ni_nid.nid_addr, addr, + ifaces[if_idx].li_size) == 0) break; } } @@ -1673,11 +1671,13 @@ int lnet_inet_select(struct lnet_ni *ni, return if_idx; if (ni->ni_interface) - CERROR("ksocklnd: failed to find interface %s%s%s\n", + CERROR("%s: failed to find interface %s%s%s\n", + libcfs_lnd2modname(ni->ni_nid.nid_type), ni->ni_interface, addr_set ? "@" : "", addr_set ? libcfs_nidstr(&ni->ni_nid) : ""); else - CERROR("ksocklnd: failed to find IP address %s\n", + CERROR("%s: failed to find IP address %s\n", + libcfs_lnd2modname(ni->ni_nid.nid_type), libcfs_nidstr(&ni->ni_nid)); return -EINVAL; @@ -1719,7 +1719,7 @@ lnet_parse_ip2nets(const char **networksp, const char *ip2nets) } for (i = 0; i < nip; i++) - ipaddrs[i] = ifaces[i].li_ipaddr; + ipaddrs[i] = ntohl(ifaces[i].li_ipaddr); rc = lnet_match_networks(networksp, ip2nets, ipaddrs, nip); if (rc < 0) { -- 1.8.3.1