From: Chris Horn Date: Wed, 11 Sep 2024 16:42:24 +0000 (-0600) Subject: LU-14288 nodemap: Use nidmasks for IPv6 NIDs X-Git-Tag: 2.16.51~85 X-Git-Url: https://git.whamcloud.com/?a=commitdiff_plain;h=8f6988be4416fe0256583e5d6ea63e92ec0811ca;p=fs%2Flustre-release.git LU-14288 nodemap: Use nidmasks for IPv6 NIDs Using nidmasks for nodemap ranges allows us to specify more than one IPv6 NID in a range. In range_create(), we construct a string representing a nidmask based on the specified NID and netmask. This is passed to cfs_parse_nidlist(), and the resulting nidlist is stored in lu_nid_range::rn_nidlist. cfs_match_nid() is used in range_find() and range_search() to locate appropriate ranges. Test-Parameters: trivial Test-Parameters: testlist=sanity-sec env=FORCE_LARGE_NID=true,LOAD_MODULES_REMOTE=true Signed-off-by: Chris Horn Change-Id: Ib1f60eda6fbc9d0214b6c63dcc8656eab3977a8b Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/56184 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Serguei Smirnov Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- diff --git a/libcfs/libcfs/util/nidstrings.c b/libcfs/libcfs/util/nidstrings.c index 5f6d9c9..eab4cff 100644 --- a/libcfs/libcfs/util/nidstrings.c +++ b/libcfs/libcfs/util/nidstrings.c @@ -1662,7 +1662,8 @@ cfs_parse_nidlist(char *orig, int len, struct list_head *nidlist) } static int -match_nidmask(struct lnet_nid *nid, struct nidmask *nm, struct netstrfns *nf) +match_nidmask(const struct lnet_nid *nid, struct nidmask *nm, + struct netstrfns *nf) { __be32 addr[4] = { nid->nid_addr[0], nid->nid_addr[1], nid->nid_addr[2], nid->nid_addr[3] }; @@ -1693,7 +1694,7 @@ match_nidmask(struct lnet_nid *nid, struct nidmask *nm, struct netstrfns *nf) * \retval 1 on match * \retval 0 otherwises */ -int cfs_match_nid(struct lnet_nid *nid, struct list_head *nidlist) +int cfs_match_nid(const struct lnet_nid *nid, struct list_head *nidlist) { struct nidrange *nr; struct nidmask *nm; diff --git a/lnet/include/uapi/linux/lnet/nidstr.h b/lnet/include/uapi/linux/lnet/nidstr.h index fe4c209..d67f1d1 100644 --- a/lnet/include/uapi/linux/lnet/nidstr.h +++ b/lnet/include/uapi/linux/lnet/nidstr.h @@ -95,6 +95,9 @@ static inline char *libcfs_nidstr(const struct lnet_nid *nid) int libcfs_strnid(struct lnet_nid *nid, const char *str); #ifdef __KERNEL__ char *libcfs_idstr(struct lnet_processid *id); +u8 cfs_nidmask_get_length(struct list_head *nidlist); +int cfs_nidmask_get_base_nidstr(char *buf, int count, + struct list_head *nidlist); #endif __u32 libcfs_str2net(const char *str); lnet_nid_t libcfs_str2nid(const char *str); @@ -105,7 +108,7 @@ char *libcfs_id2str(struct lnet_process_id id); void cfs_free_nidlist(struct list_head *list); int cfs_parse_nidlist(char *str, int len, struct list_head *list); int cfs_print_nidlist(char *buffer, int count, struct list_head *list); -int cfs_match_nid(struct lnet_nid *nid, struct list_head *list); +int cfs_match_nid(const struct lnet_nid *nid, struct list_head *list); int cfs_match_net(__u32 net_id, __u32 net_type, struct list_head *net_num_list); diff --git a/lnet/lnet/nidstrings.c b/lnet/lnet/nidstrings.c index b960725..b8c4507 100644 --- a/lnet/lnet/nidstrings.c +++ b/lnet/lnet/nidstrings.c @@ -737,12 +737,14 @@ cfs_parse_nidlist(char *orig, int len, struct list_head *nidlist) cfs_free_nidlist(nidlist); else if (list_empty(nidlist)) rc = -EINVAL; + return rc; } EXPORT_SYMBOL(cfs_parse_nidlist); static int -match_nidmask(struct lnet_nid *nid, struct nidmask *nm, struct netstrfns *nf) +match_nidmask(const struct lnet_nid *nid, struct nidmask *nm, + struct netstrfns *nf) { __be32 addr[4] = { nid->nid_addr[0], nid->nid_addr[1], nid->nid_addr[2], nid->nid_addr[3] }; @@ -773,7 +775,7 @@ match_nidmask(struct lnet_nid *nid, struct nidmask *nm, struct netstrfns *nf) * \retval 1 on match * \retval 0 otherwises */ -int cfs_match_nid(struct lnet_nid *nid, struct list_head *nidlist) +int cfs_match_nid(const struct lnet_nid *nid, struct list_head *nidlist) { struct nidrange *nr; struct nidmask *nm; @@ -843,36 +845,47 @@ cfs_print_addrranges(char *buffer, int count, struct list_head *addrranges, } static int +print_nidmask(char *buffer, int count, struct nidmask *nm, struct nidrange *nr, + bool omit_prefix) +{ + int i = 0; + struct sockaddr_storage sa = {}; + u8 max; + + /* parse_nidmask() ensures nm_family is set to either AF_INET + * or AF_INET6 + */ + sa.ss_family = nm->nm_family; + if (nm->nm_family == AF_INET) { + memcpy(&((struct sockaddr_in *)(&sa))->sin_addr.s_addr, + &nm->nm_addr.ipv4, sizeof(struct in_addr)); + max = 32; + } else { + memcpy(&((struct sockaddr_in6 *)(&sa))->sin6_addr.s6_addr, + &nm->nm_addr.ipv6, sizeof(struct in6_addr)); + max = 128; + } + i += rpc_ntop((struct sockaddr *)&sa, buffer + i, count - i); + if (nm->nm_prefix_len < max && !omit_prefix) + i += scnprintf(buffer + i, count - i, "/%u", + nm->nm_prefix_len); + i += cfs_print_network(buffer + i, count - i, nr); + + return i; +} + +static int cfs_print_nidmasks(char *buffer, int count, struct list_head *nidmasks, struct nidrange *nr) { int i = 0; struct nidmask *nm; - struct sockaddr_storage sa = {}; - u8 max; list_for_each_entry(nm, nidmasks, nm_link) { if (i != 0) i += scnprintf(buffer + i, count - i, " "); - /* parse_nidmask() ensures nm_family is set to either AF_INET - * or AF_INET6 - */ - sa.ss_family = nm->nm_family; - if (nm->nm_family == AF_INET) { - memcpy(&((struct sockaddr_in *)(&sa))->sin_addr.s_addr, - &nm->nm_addr.ipv4, sizeof(struct in_addr)); - max = 32; - } else { - memcpy(&((struct sockaddr_in6 *)(&sa))->sin6_addr.s6_addr, - &nm->nm_addr.ipv6, sizeof(struct in6_addr)); - max = 128; - } - i += rpc_ntop((struct sockaddr *)&sa, buffer + i, count - i); - if (nm->nm_prefix_len < max) - i += scnprintf(buffer + i, count - i, "/%u", - nm->nm_prefix_len); - i += cfs_print_network(buffer + i, count - i, nr); + i += print_nidmask(buffer + i, count - i, nm, nr, false); } return i; @@ -923,6 +936,47 @@ int cfs_print_nidlist(char *buffer, int count, struct list_head *nidlist) } EXPORT_SYMBOL(cfs_print_nidlist); +/* Caller should provide a nidlist with a single nidmask */ +u8 +cfs_nidmask_get_length(struct list_head *nidlist) +{ + struct nidrange *nr; + struct nidmask *nm; + u8 len = 0; + + list_for_each_entry(nr, nidlist, nr_link) { + list_for_each_entry(nm, &nr->nr_nidmasks, nm_link) { + if (len) + return 0; + len = nm->nm_prefix_len; + } + } + return len; +} +EXPORT_SYMBOL(cfs_nidmask_get_length); + +/* Caller should provide a nidlist with a single nidmask */ +int +cfs_nidmask_get_base_nidstr(char *buf, int count, struct list_head *nidlist) +{ + struct nidrange *nr; + struct nidmask *nm; + bool found = false; + + list_for_each_entry(nr, nidlist, nr_link) { + list_for_each_entry(nm, &nr->nr_nidmasks, nm_link) { + if (found) + return -EINVAL; + + print_nidmask(buf, count, nm, nr, true); + found = true; + } + } + + return 0; +} +EXPORT_SYMBOL(cfs_nidmask_get_base_nidstr); + static int libcfs_lo_str2addr(const char *str, int nob, __u32 *addr) { diff --git a/lustre/doc/lctl-nodemap-add-range.8 b/lustre/doc/lctl-nodemap-add-range.8 index ecf15c2..0a8f1dc 100644 --- a/lustre/doc/lctl-nodemap-add-range.8 +++ b/lustre/doc/lctl-nodemap-add-range.8 @@ -26,24 +26,32 @@ with the added constraint that the range must be contiguous. .SH Formal LNet NID Range Definition .EX NID_RANGE :== ADDR_RANGE@NET -ADDR_RANGE :== '*' | IPADDR_RANGE | NUM_RANGE +ADDR_RANGE :== '*' | NETMASK | IPV6_ADDR | IPADDR_RANGE | NUM_RANGE IPADDR_RANGE :== NUM_RANGE.NUM_RANGE.NUM_RANGE.NUM_RANGE NUM_RANGE :== NUMBER | EXPR_LIST EXPR_LIST :== '['RANGE_EXPR[,RANGE_EXPR]']' -RANGE_EXPR :== NUMBER | NUMBER-NUMBER | NUMBER-NUMBER/NUMBER +RANGE_EXPR :== NUMBER | MIN-MAX | MIN-MAX/SKIP NET :== NET_TYPE[NUMBER] .\" Currently supported LNet types are listed in libcfs_netstrfns[] NET_TYPE :== "lo" | "tcp" | "o2ib" | "ptlf" | "gni" | "gip" | "kfi" NUMBER :== NONNEGATIVE_DECIMAL | HEXADECIMAL .EE Where +.IR NETMASK +is an IPv4 or IPv6 network mask in CIDR notation, +.IR IPV6_ADDR +is a single IPv6 address, .IR MIN - MAX / SKIP indicates a sequence of numbers starting at .I MIN and incrementing by .I SKIP each time until no larger than -.IR MAX . +.IR MAX , +and +.IR MIN - MAX +is equivalent to +.IR MIN - MAX / 1 . .SH EXAMPLES .EX .B # lctl nodemap_add_range --name remotesite --range 192.168.1.[1-254]@tcp diff --git a/lustre/doc/lctl-nodemap-del-range.8 b/lustre/doc/lctl-nodemap-del-range.8 index da91668..7fcf1a8 100644 --- a/lustre/doc/lctl-nodemap-del-range.8 +++ b/lustre/doc/lctl-nodemap-del-range.8 @@ -20,7 +20,9 @@ The NID range that should be deleted from the nodemap. .SH EXAMPLES .EX .B # lctl nodemap_del_range --name remotesite --range 192.168.1.[1-254]@tcp -.B # lctl nodemap_del_range --name othersite --range 192.168.2.[1-254]@tcp +.B # lctl nodemap_del_range --name secondremotesite --range 192.168.2.[1-254]@tcp +.B # lctl nodemap_del_range --name thirdremotesite --range 2001:0db8::/32@tcp +.B # lctl nodemap_del_range --name fourthremotesite --range [1-60000]@kfi .EE .SH AVAILABILITY .B lctl nodemap_del_range diff --git a/lustre/ptlrpc/nodemap_handler.c b/lustre/ptlrpc/nodemap_handler.c index 3571cbf..c9c43ed 100644 --- a/lustre/ptlrpc/nodemap_handler.c +++ b/lustre/ptlrpc/nodemap_handler.c @@ -330,22 +330,29 @@ int nodemap_parse_range(const char *range_str, struct lnet_nid range[2], char *start_nidstr; char *end_nidstr; int rc = 0; + LIST_HEAD(nidlist); snprintf(buf, sizeof(buf), "%s", range_str); ptr = buf; - /* For large NID we use netmasks. Currently we only - * support /128 which is a single NID. - */ - if (strchr(ptr, '/')) { - start_nidstr = strsep(&ptr, "/"); + /* For large NIDs we interpret range_str as a nidmask */ + if (!cfs_parse_nidlist(buf, strlen(buf), &nidlist)) { + *netmask = cfs_nidmask_get_length(&nidlist); + if (!*netmask) + GOTO(out, rc = -EINVAL); - rc = kstrtou8(ptr, 10, netmask); - if (rc < 0) - GOTO(out, rc); - if (*netmask != 128) - GOTO(out, rc = -ERANGE); - end_nidstr = start_nidstr; + rc = cfs_nidmask_get_base_nidstr(buf, sizeof(buf), &nidlist); + if (rc) { + cfs_free_nidlist(&nidlist); + GOTO(out, rc = -EINVAL); + } + + end_nidstr = start_nidstr = buf; + + cfs_free_nidlist(&nidlist); + + CDEBUG(D_INFO, "nidstr: %s netmask: %u\n", + start_nidstr, *netmask); } else { start_nidstr = strsep(&ptr, ":"); end_nidstr = strsep(&ptr, ":"); diff --git a/lustre/ptlrpc/nodemap_internal.h b/lustre/ptlrpc/nodemap_internal.h index 7abd2088..a3be687 100644 --- a/lustre/ptlrpc/nodemap_internal.h +++ b/lustre/ptlrpc/nodemap_internal.h @@ -66,6 +66,10 @@ struct lu_nid_range { lnet_nid_t rn_subtree_last; /* Large NID netmask */ u8 rn_netmask; + /* The nidlist corresponding to the nidrange constructed from + * rn_start and rn_netmask + */ + struct list_head rn_nidlist; struct rb_node rn_rb; }; diff --git a/lustre/ptlrpc/nodemap_range.c b/lustre/ptlrpc/nodemap_range.c index f71130b..c351f86 100644 --- a/lustre/ptlrpc/nodemap_range.c +++ b/lustre/ptlrpc/nodemap_range.c @@ -49,9 +49,12 @@ INTERVAL_TREE_DEFINE(struct lu_nid_range, rn_rb, lnet_nid_t, rn_subtree_last, /* * range constructor * - * \param min starting nid of the range - * \param max ending nid of the range + * \param config nodemap config - used to set range id + * \param start_nid starting nid of the range + * \param end_nid ending nid of the range + * \param netmask network mask prefix length * \param nodemap nodemap that contains this range + * \param range_id should be 0 unless loading from disk * \retval lu_nid_range on success, NULL on failure */ struct lu_nid_range *range_create(struct nodemap_config *config, @@ -62,6 +65,7 @@ struct lu_nid_range *range_create(struct nodemap_config *config, { struct nodemap_range_tree *nm_range_tree; struct lu_nid_range *range; + LIST_HEAD(tmp_nidlist); if (LNET_NID_NET(start_nid) != LNET_NID_NET(end_nid)) return NULL; @@ -75,10 +79,58 @@ struct lu_nid_range *range_create(struct nodemap_config *config, if (LNET_NIDADDR(nid4[0]) > LNET_NIDADDR(nid4[1])) return NULL; } else if (!nid_same(start_nid, end_nid)) { - /* FIXME Currently we only support one large NID per nodemap */ + /* A netmask is used with start_nid to form a nidmask. If the + * start_nid and end_nid differ then this indicates the + * specified range was malformed + */ return NULL; } + if (netmask) { + /* +4 for '/' */ + char nidstr[LNET_NIDSTR_SIZE + 4]; + char net[LNET_NIDSTR_SIZE]; + char *c; + int rc; + + if (netmask > 999) { + /* If the netmask is somehow more than three characters + * then the logic below could truncate it which could + * result in creating a valid netmask value from bad + * input. + * cfs_parse_nidlist() will check whether the netmask + * is valid for the address type + */ + CERROR("Invalid netmask %u\n", netmask); + return NULL; + } + + /* nidstr = @ */ + snprintf(nidstr, sizeof(nidstr), "%s", + libcfs_nidstr(start_nid)); + + c = strchr(nidstr, '@'); + + /* net = @ */ + strscpy(net, c, sizeof(net)); + + *c = '\0'; + + /* nidstr = / */ + snprintf(c, sizeof(nidstr) - strlen(nidstr), "/%u", netmask); + + /* nidstr = /@ + * (-1 to ensure room for null byte) + */ + strncat(nidstr, net, sizeof(nidstr) - strlen(nidstr) - 1); + + rc = cfs_parse_nidlist(nidstr, strlen(nidstr), &tmp_nidlist); + if (rc) { + CERROR("Invalid nidmask %s rc = %d\n", nidstr, rc); + return NULL; + } + } + OBD_ALLOC_PTR(range); if (range == NULL) { CERROR("cannot allocate lu_nid_range of size %zu bytes\n", @@ -103,6 +155,9 @@ struct lu_nid_range *range_create(struct nodemap_config *config, range->rn_end = *end_nid; INIT_LIST_HEAD(&range->rn_list); + INIT_LIST_HEAD(&range->rn_nidlist); + if (!list_empty(&tmp_nidlist)) + list_splice(&tmp_nidlist, &range->rn_nidlist); return range; } @@ -143,15 +198,15 @@ struct lu_nid_range *range_find(struct nodemap_config *config, if (!list_empty(&config->nmc_netmask_setup)) { struct lu_nid_range *range_temp; + u8 len; - /* FIXME. We scan the config for large NIDs. Each range - * only contains one large NID for now. - */ list_for_each_entry_safe(range, range_temp, &config->nmc_netmask_setup, rn_collect) { - if (nid_same(&range->rn_start, start_nid) && - range->rn_netmask == netmask) + len = cfs_nidmask_get_length(&range->rn_nidlist); + + if (cfs_match_nid(start_nid, &range->rn_nidlist) && + netmask == len) return range; } } @@ -165,6 +220,8 @@ struct lu_nid_range *range_find(struct nodemap_config *config, void range_destroy(struct lu_nid_range *range) { LASSERT(list_empty(&range->rn_list) == 0); + if (!list_empty(&range->rn_nidlist)) + cfs_free_nidlist(&range->rn_nidlist); OBD_FREE_PTR(range); } @@ -172,7 +229,7 @@ void range_destroy(struct lu_nid_range *range) /* * insert an nid range into the interval tree * - * \param range range to insetr + * \param range range to insert * \retval 0 on success * * This function checks that the given nid range @@ -193,6 +250,10 @@ int range_insert(struct nodemap_config *config, struct lu_nid_range *range) nm_range_insert(range, &nm_range_tree->nmrt_range_interval_root); } else { + if (range_find(config, &range->rn_start, &range->rn_end, + range->rn_netmask)) + return -EEXIST; + list_add(&range->rn_collect, &config->nmc_netmask_setup); } return 0; @@ -220,7 +281,7 @@ void range_delete(struct nodemap_config *config, struct lu_nid_range *range) } /* - * search the interval tree for an nid within a range + * search the interval tree for a nid within a range * * \param nid nid to search for */ @@ -234,16 +295,15 @@ struct lu_nid_range *range_search(struct nodemap_config *config, return nm_range_iter_first(&nm_range_tree->nmrt_range_interval_root, lnet_nid_to_nid4(nid), lnet_nid_to_nid4(nid)); - } else if (!list_empty(&config->nmc_netmask_setup)) { + } + + if (!list_empty(&config->nmc_netmask_setup)) { struct lu_nid_range *range, *range_temp; - /* FIXME. We scan the config for the large NIDs. Each range - * only contains one large NID for now. - */ list_for_each_entry_safe(range, range_temp, &config->nmc_netmask_setup, rn_collect) { - if (nid_same(&range->rn_start, nid)) + if (cfs_match_nid(nid, &range->rn_nidlist)) return range; } } diff --git a/lustre/tests/sanity-sec.sh b/lustre/tests/sanity-sec.sh index 81661a2..01177a8 100755 --- a/lustre/tests/sanity-sec.sh +++ b/lustre/tests/sanity-sec.sh @@ -1214,7 +1214,7 @@ create_fops_nodemaps() { for client in $clients; do local client_ip=$(host_nids_address $client $NETTYPE) local client_nid=$(h2nettype $client_ip) - [[ "$client_nid" =~ ":" ]] && client_nid+="/128" + do_facet mgs $LCTL nodemap_add c${i} || return 1 do_facet mgs $LCTL nodemap_add_range \ --name c${i} --range $client_nid || { @@ -5073,10 +5073,10 @@ test_55() { client_ip=$(host_nids_address $HOSTNAME $NETTYPE) client_nid=$(h2nettype $client_ip) - [[ "$client_nid" =~ ":" ]] && client_nid+="/128" do_facet mgs $LCTL nodemap_add c0 do_facet mgs $LCTL nodemap_add_range \ - --name c0 --range $client_nid + --name c0 --range $client_nid || + error "Add range $client_nid to c0 failed rc = $?" do_facet mgs $LCTL nodemap_modify --name c0 \ --property admin --value 0 do_facet mgs $LCTL nodemap_modify --name c0 \ @@ -5519,7 +5519,6 @@ setup_61() { client_ip=$(host_nids_address $HOSTNAME $NETTYPE) client_nid=$(h2nettype $client_ip) - [[ "$client_nid" =~ ":" ]] && client_nid+="/128" do_facet mgs $LCTL nodemap_add c0 do_facet mgs $LCTL nodemap_add_range \ --name c0 --range $client_nid || { @@ -5831,10 +5830,10 @@ setup_64() { client_ip=$(host_nids_address $HOSTNAME $NETTYPE) client_nid=$(h2nettype $client_ip) - [[ "$client_nid" =~ ":" ]] && client_nid+="/128" do_facet mgs $LCTL nodemap_add c0 do_facet mgs $LCTL nodemap_add_range \ - --name c0 --range $client_nid + --name c0 --range $client_nid || + error "Add range $client_nid to c0 failed rc = $?" do_facet mgs $LCTL nodemap_modify --name c0 \ --property admin --value 1 do_facet mgs $LCTL nodemap_modify --name c0 \ diff --git a/lustre/utils/obd.c b/lustre/utils/obd.c index d1599f3..92ab07b 100644 --- a/lustre/utils/obd.c +++ b/lustre/utils/obd.c @@ -4219,25 +4219,11 @@ static int parse_nid_range(char *nodemap_range, char *nid_range, int range_len) char min_nid[LNET_NIDSTR_SIZE + 1]; char max_nid[LNET_NIDSTR_SIZE + 1]; struct list_head nidlist; - char *netmask = NULL; int rc = 0; - netmask = strchr(nodemap_range, '/'); - if (netmask) { - unsigned long mask; - - /* FIXME !!! Only 128 netmask is supported. This means - * nodemap will only support one large NID. - */ - mask = strtoul(++netmask, NULL, 10); - if (mask < 0) - return -errno; - - if (mask != 128) - return -ERANGE; - + if (strchr(nodemap_range, '/') || strchr(nodemap_range, ':')) { strncpy(nid_range, nodemap_range, range_len); - return rc; + return 0; } INIT_LIST_HEAD(&nidlist);