Whamcloud - gitweb
LU-14288 nodemap: Use nidmasks for IPv6 NIDs 84/56184/10
authorChris Horn <chris.horn@hpe.com>
Wed, 11 Sep 2024 16:42:24 +0000 (10:42 -0600)
committerOleg Drokin <green@whamcloud.com>
Mon, 16 Dec 2024 08:01:37 +0000 (08:01 +0000)
Using nidmasks for nodemap ranges allows us to specify more than one
IPv6 NID in a range. In range_create(), we construct a string
representing a nidmask based on the specified NID and netmask. This
is passed to cfs_parse_nidlist(), and the resulting nidlist is stored
in lu_nid_range::rn_nidlist. cfs_match_nid() is used in range_find()
and range_search() to locate appropriate ranges.

Test-Parameters: trivial
Test-Parameters: testlist=sanity-sec env=FORCE_LARGE_NID=true,LOAD_MODULES_REMOTE=true
Signed-off-by: Chris Horn <chris.horn@hpe.com>
Change-Id: Ib1f60eda6fbc9d0214b6c63dcc8656eab3977a8b
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/56184
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
libcfs/libcfs/util/nidstrings.c
lnet/include/uapi/linux/lnet/nidstr.h
lnet/lnet/nidstrings.c
lustre/doc/lctl-nodemap-add-range.8
lustre/doc/lctl-nodemap-del-range.8
lustre/ptlrpc/nodemap_handler.c
lustre/ptlrpc/nodemap_internal.h
lustre/ptlrpc/nodemap_range.c
lustre/tests/sanity-sec.sh
lustre/utils/obd.c

index 5f6d9c9..eab4cff 100644 (file)
@@ -1662,7 +1662,8 @@ cfs_parse_nidlist(char *orig, int len, struct list_head *nidlist)
 }
 
 static int
-match_nidmask(struct lnet_nid *nid, struct nidmask *nm, struct netstrfns *nf)
+match_nidmask(const struct lnet_nid *nid, struct nidmask *nm,
+             struct netstrfns *nf)
 {
        __be32 addr[4] = { nid->nid_addr[0], nid->nid_addr[1],
                           nid->nid_addr[2], nid->nid_addr[3] };
@@ -1693,7 +1694,7 @@ match_nidmask(struct lnet_nid *nid, struct nidmask *nm, struct netstrfns *nf)
  * \retval 1 on match
  * \retval 0  otherwises
  */
-int cfs_match_nid(struct lnet_nid *nid, struct list_head *nidlist)
+int cfs_match_nid(const struct lnet_nid *nid, struct list_head *nidlist)
 {
        struct nidrange *nr;
        struct nidmask *nm;
index fe4c209..d67f1d1 100644 (file)
@@ -95,6 +95,9 @@ static inline char *libcfs_nidstr(const struct lnet_nid *nid)
 int libcfs_strnid(struct lnet_nid *nid, const char *str);
 #ifdef __KERNEL__
 char *libcfs_idstr(struct lnet_processid *id);
+u8 cfs_nidmask_get_length(struct list_head *nidlist);
+int cfs_nidmask_get_base_nidstr(char *buf, int count,
+                               struct list_head *nidlist);
 #endif
 __u32 libcfs_str2net(const char *str);
 lnet_nid_t libcfs_str2nid(const char *str);
@@ -105,7 +108,7 @@ char *libcfs_id2str(struct lnet_process_id id);
 void cfs_free_nidlist(struct list_head *list);
 int cfs_parse_nidlist(char *str, int len, struct list_head *list);
 int cfs_print_nidlist(char *buffer, int count, struct list_head *list);
-int cfs_match_nid(struct lnet_nid *nid, struct list_head *list);
+int cfs_match_nid(const struct lnet_nid *nid, struct list_head *list);
 int cfs_match_net(__u32 net_id, __u32 net_type,
                  struct list_head *net_num_list);
 
index b960725..b8c4507 100644 (file)
@@ -737,12 +737,14 @@ cfs_parse_nidlist(char *orig, int len, struct list_head *nidlist)
                cfs_free_nidlist(nidlist);
        else if (list_empty(nidlist))
                rc = -EINVAL;
+
        return rc;
 }
 EXPORT_SYMBOL(cfs_parse_nidlist);
 
 static int
-match_nidmask(struct lnet_nid *nid, struct nidmask *nm, struct netstrfns *nf)
+match_nidmask(const struct lnet_nid *nid, struct nidmask *nm,
+             struct netstrfns *nf)
 {
        __be32 addr[4] = { nid->nid_addr[0], nid->nid_addr[1],
                           nid->nid_addr[2], nid->nid_addr[3] };
@@ -773,7 +775,7 @@ match_nidmask(struct lnet_nid *nid, struct nidmask *nm, struct netstrfns *nf)
  * \retval 1 on match
  * \retval 0  otherwises
  */
-int cfs_match_nid(struct lnet_nid *nid, struct list_head *nidlist)
+int cfs_match_nid(const struct lnet_nid *nid, struct list_head *nidlist)
 {
        struct nidrange *nr;
        struct nidmask *nm;
@@ -843,36 +845,47 @@ cfs_print_addrranges(char *buffer, int count, struct list_head *addrranges,
 }
 
 static int
+print_nidmask(char *buffer, int count, struct nidmask *nm, struct nidrange *nr,
+             bool omit_prefix)
+{
+       int i = 0;
+       struct sockaddr_storage sa = {};
+       u8 max;
+
+       /* parse_nidmask() ensures nm_family is set to either AF_INET
+        * or AF_INET6
+        */
+       sa.ss_family = nm->nm_family;
+       if (nm->nm_family == AF_INET) {
+               memcpy(&((struct sockaddr_in *)(&sa))->sin_addr.s_addr,
+                      &nm->nm_addr.ipv4, sizeof(struct in_addr));
+               max = 32;
+       } else {
+               memcpy(&((struct sockaddr_in6 *)(&sa))->sin6_addr.s6_addr,
+                      &nm->nm_addr.ipv6, sizeof(struct in6_addr));
+               max = 128;
+       }
+       i += rpc_ntop((struct sockaddr *)&sa, buffer + i, count - i);
+       if (nm->nm_prefix_len < max && !omit_prefix)
+               i += scnprintf(buffer + i, count - i, "/%u",
+                              nm->nm_prefix_len);
+       i += cfs_print_network(buffer + i, count - i, nr);
+
+       return i;
+}
+
+static int
 cfs_print_nidmasks(char *buffer, int count, struct list_head *nidmasks,
                   struct nidrange *nr)
 {
        int i = 0;
        struct nidmask *nm;
-       struct sockaddr_storage sa = {};
-       u8 max;
 
        list_for_each_entry(nm, nidmasks, nm_link) {
                if (i != 0)
                        i += scnprintf(buffer + i, count - i, " ");
 
-               /* parse_nidmask() ensures nm_family is set to either AF_INET
-                * or AF_INET6
-                */
-               sa.ss_family = nm->nm_family;
-               if (nm->nm_family == AF_INET) {
-                       memcpy(&((struct sockaddr_in *)(&sa))->sin_addr.s_addr,
-                              &nm->nm_addr.ipv4, sizeof(struct in_addr));
-                       max = 32;
-               } else {
-                       memcpy(&((struct sockaddr_in6 *)(&sa))->sin6_addr.s6_addr,
-                              &nm->nm_addr.ipv6, sizeof(struct in6_addr));
-                       max = 128;
-               }
-               i += rpc_ntop((struct sockaddr *)&sa, buffer + i, count - i);
-               if (nm->nm_prefix_len < max)
-                       i += scnprintf(buffer + i, count - i, "/%u",
-                                      nm->nm_prefix_len);
-               i += cfs_print_network(buffer + i, count - i, nr);
+               i += print_nidmask(buffer + i, count - i, nm, nr, false);
        }
 
        return i;
@@ -923,6 +936,47 @@ int cfs_print_nidlist(char *buffer, int count, struct list_head *nidlist)
 }
 EXPORT_SYMBOL(cfs_print_nidlist);
 
+/* Caller should provide a nidlist with a single nidmask */
+u8
+cfs_nidmask_get_length(struct list_head *nidlist)
+{
+       struct nidrange *nr;
+       struct nidmask *nm;
+       u8 len = 0;
+
+       list_for_each_entry(nr, nidlist, nr_link) {
+               list_for_each_entry(nm, &nr->nr_nidmasks, nm_link) {
+                       if (len)
+                               return 0;
+                       len = nm->nm_prefix_len;
+               }
+       }
+       return len;
+}
+EXPORT_SYMBOL(cfs_nidmask_get_length);
+
+/* Caller should provide a nidlist with a single nidmask */
+int
+cfs_nidmask_get_base_nidstr(char *buf, int count, struct list_head *nidlist)
+{
+       struct nidrange *nr;
+       struct nidmask *nm;
+       bool found = false;
+
+       list_for_each_entry(nr, nidlist, nr_link) {
+               list_for_each_entry(nm, &nr->nr_nidmasks, nm_link) {
+                       if (found)
+                               return -EINVAL;
+
+                       print_nidmask(buf, count, nm, nr, true);
+                       found = true;
+               }
+       }
+
+       return 0;
+}
+EXPORT_SYMBOL(cfs_nidmask_get_base_nidstr);
+
 static int
 libcfs_lo_str2addr(const char *str, int nob, __u32 *addr)
 {
index ecf15c2..0a8f1dc 100644 (file)
@@ -26,24 +26,32 @@ with the added constraint that the range must be contiguous.
 .SH Formal LNet NID Range Definition
 .EX
 NID_RANGE    :== ADDR_RANGE@NET
-ADDR_RANGE   :== '*' | IPADDR_RANGE | NUM_RANGE
+ADDR_RANGE   :== '*' | NETMASK | IPV6_ADDR | IPADDR_RANGE | NUM_RANGE
 IPADDR_RANGE :== NUM_RANGE.NUM_RANGE.NUM_RANGE.NUM_RANGE
 NUM_RANGE    :== NUMBER | EXPR_LIST
 EXPR_LIST    :== '['RANGE_EXPR[,RANGE_EXPR]']'
-RANGE_EXPR   :== NUMBER | NUMBER-NUMBER | NUMBER-NUMBER/NUMBER
+RANGE_EXPR   :== NUMBER | MIN-MAX | MIN-MAX/SKIP
 NET          :== NET_TYPE[NUMBER]
 .\" Currently supported LNet types are listed in libcfs_netstrfns[]
 NET_TYPE     :== "lo" | "tcp" | "o2ib" | "ptlf" | "gni" | "gip" | "kfi"
 NUMBER       :== NONNEGATIVE_DECIMAL | HEXADECIMAL
 .EE
 Where
+.IR NETMASK
+is an IPv4 or IPv6 network mask in CIDR notation,
+.IR IPV6_ADDR
+is a single IPv6 address,
 .IR MIN - MAX / SKIP
 indicates a sequence of numbers starting at
 .I MIN
 and incrementing by
 .I SKIP
 each time until no larger than
-.IR MAX .
+.IR MAX ,
+and
+.IR MIN - MAX
+is equivalent to
+.IR MIN - MAX / 1 .
 .SH EXAMPLES
 .EX
 .B # lctl nodemap_add_range --name remotesite --range 192.168.1.[1-254]@tcp
index da91668..7fcf1a8 100644 (file)
@@ -20,7 +20,9 @@ The NID range that should be deleted from the nodemap.
 .SH EXAMPLES
 .EX
 .B # lctl nodemap_del_range --name remotesite --range 192.168.1.[1-254]@tcp
-.B # lctl nodemap_del_range --name othersite --range 192.168.2.[1-254]@tcp
+.B # lctl nodemap_del_range --name secondremotesite --range 192.168.2.[1-254]@tcp
+.B # lctl nodemap_del_range --name thirdremotesite --range 2001:0db8::/32@tcp
+.B # lctl nodemap_del_range --name fourthremotesite --range [1-60000]@kfi
 .EE
 .SH AVAILABILITY
 .B lctl nodemap_del_range
index 3571cbf..c9c43ed 100644 (file)
@@ -330,22 +330,29 @@ int nodemap_parse_range(const char *range_str, struct lnet_nid range[2],
        char    *start_nidstr;
        char    *end_nidstr;
        int     rc = 0;
+       LIST_HEAD(nidlist);
 
        snprintf(buf, sizeof(buf), "%s", range_str);
        ptr = buf;
 
-       /* For large NID we use netmasks. Currently we only
-        * support /128 which is a single NID.
-        */
-       if (strchr(ptr, '/')) {
-               start_nidstr = strsep(&ptr, "/");
+       /* For large NIDs we interpret range_str as a nidmask */
+       if (!cfs_parse_nidlist(buf, strlen(buf), &nidlist)) {
+               *netmask = cfs_nidmask_get_length(&nidlist);
+               if (!*netmask)
+                       GOTO(out, rc = -EINVAL);
 
-               rc = kstrtou8(ptr, 10, netmask);
-               if (rc < 0)
-                       GOTO(out, rc);
-               if (*netmask != 128)
-                       GOTO(out, rc = -ERANGE);
-               end_nidstr = start_nidstr;
+               rc = cfs_nidmask_get_base_nidstr(buf, sizeof(buf), &nidlist);
+               if (rc) {
+                       cfs_free_nidlist(&nidlist);
+                       GOTO(out, rc = -EINVAL);
+               }
+
+               end_nidstr = start_nidstr = buf;
+
+               cfs_free_nidlist(&nidlist);
+
+               CDEBUG(D_INFO, "nidstr: %s netmask: %u\n",
+                      start_nidstr, *netmask);
        } else {
                start_nidstr = strsep(&ptr, ":");
                end_nidstr = strsep(&ptr, ":");
index 7abd208..a3be687 100644 (file)
@@ -66,6 +66,10 @@ struct lu_nid_range {
        lnet_nid_t               rn_subtree_last;
        /* Large NID netmask */
        u8                       rn_netmask;
+       /* The nidlist corresponding to the nidrange constructed from
+        * rn_start and rn_netmask
+        */
+       struct list_head         rn_nidlist;
        struct rb_node           rn_rb;
 };
 
index f71130b..c351f86 100644 (file)
@@ -49,9 +49,12 @@ INTERVAL_TREE_DEFINE(struct lu_nid_range, rn_rb, lnet_nid_t, rn_subtree_last,
 /*
  * range constructor
  *
- * \param      min             starting nid of the range
- * \param      max             ending nid of the range
+ * \param      config          nodemap config - used to set range id
+ * \param      start_nid       starting nid of the range
+ * \param      end_nid         ending nid of the range
+ * \param      netmask         network mask prefix length
  * \param      nodemap         nodemap that contains this range
+ * \param      range_id        should be 0 unless loading from disk
  * \retval     lu_nid_range on success, NULL on failure
  */
 struct lu_nid_range *range_create(struct nodemap_config *config,
@@ -62,6 +65,7 @@ struct lu_nid_range *range_create(struct nodemap_config *config,
 {
        struct nodemap_range_tree *nm_range_tree;
        struct lu_nid_range *range;
+       LIST_HEAD(tmp_nidlist);
 
        if (LNET_NID_NET(start_nid) != LNET_NID_NET(end_nid))
                return NULL;
@@ -75,10 +79,58 @@ struct lu_nid_range *range_create(struct nodemap_config *config,
                if (LNET_NIDADDR(nid4[0]) > LNET_NIDADDR(nid4[1]))
                        return NULL;
        } else if (!nid_same(start_nid, end_nid)) {
-               /* FIXME Currently we only support one large NID per nodemap */
+               /* A netmask is used with start_nid to form a nidmask. If the
+                * start_nid and end_nid differ then this indicates the
+                * specified range was malformed
+                */
                return NULL;
        }
 
+       if (netmask) {
+               /* +4 for '/<prefix_length>' */
+               char nidstr[LNET_NIDSTR_SIZE + 4];
+               char net[LNET_NIDSTR_SIZE];
+               char *c;
+               int rc;
+
+               if (netmask > 999) {
+                       /* If the netmask is somehow more than three characters
+                        * then the logic below could truncate it which could
+                        * result in creating a valid netmask value from bad
+                        * input.
+                        * cfs_parse_nidlist() will check whether the netmask
+                        * is valid for the address type
+                        */
+                       CERROR("Invalid netmask %u\n", netmask);
+                       return NULL;
+               }
+
+               /* nidstr = <addr>@<net> */
+               snprintf(nidstr, sizeof(nidstr), "%s",
+                        libcfs_nidstr(start_nid));
+
+               c = strchr(nidstr, '@');
+
+               /* net = @<net> */
+               strscpy(net, c, sizeof(net));
+
+               *c = '\0';
+
+               /* nidstr = <addr>/<prefix_length> */
+               snprintf(c, sizeof(nidstr) - strlen(nidstr), "/%u", netmask);
+
+               /* nidstr = <addr>/<prefix_length>@<net>
+                * (-1 to ensure room for null byte)
+                */
+               strncat(nidstr, net, sizeof(nidstr) - strlen(nidstr) - 1);
+
+               rc = cfs_parse_nidlist(nidstr, strlen(nidstr), &tmp_nidlist);
+               if (rc) {
+                       CERROR("Invalid nidmask %s rc = %d\n", nidstr, rc);
+                       return NULL;
+               }
+       }
+
        OBD_ALLOC_PTR(range);
        if (range == NULL) {
                CERROR("cannot allocate lu_nid_range of size %zu bytes\n",
@@ -103,6 +155,9 @@ struct lu_nid_range *range_create(struct nodemap_config *config,
        range->rn_end = *end_nid;
 
        INIT_LIST_HEAD(&range->rn_list);
+       INIT_LIST_HEAD(&range->rn_nidlist);
+       if (!list_empty(&tmp_nidlist))
+               list_splice(&tmp_nidlist, &range->rn_nidlist);
 
        return range;
 }
@@ -143,15 +198,15 @@ struct lu_nid_range *range_find(struct nodemap_config *config,
 
        if (!list_empty(&config->nmc_netmask_setup)) {
                struct lu_nid_range *range_temp;
+               u8 len;
 
-               /* FIXME. We scan the config for large NIDs. Each range
-                * only contains one large NID for now.
-                */
                list_for_each_entry_safe(range, range_temp,
                                         &config->nmc_netmask_setup,
                                         rn_collect) {
-                       if (nid_same(&range->rn_start, start_nid) &&
-                           range->rn_netmask == netmask)
+                       len = cfs_nidmask_get_length(&range->rn_nidlist);
+
+                       if (cfs_match_nid(start_nid, &range->rn_nidlist) &&
+                           netmask == len)
                                return range;
                }
        }
@@ -165,6 +220,8 @@ struct lu_nid_range *range_find(struct nodemap_config *config,
 void range_destroy(struct lu_nid_range *range)
 {
        LASSERT(list_empty(&range->rn_list) == 0);
+       if (!list_empty(&range->rn_nidlist))
+               cfs_free_nidlist(&range->rn_nidlist);
 
        OBD_FREE_PTR(range);
 }
@@ -172,7 +229,7 @@ void range_destroy(struct lu_nid_range *range)
 /*
  * insert an nid range into the interval tree
  *
- * \param      range           range to insetr
+ * \param      range           range to insert
  * \retval     0 on success
  *
  * This function checks that the given nid range
@@ -193,6 +250,10 @@ int range_insert(struct nodemap_config *config, struct lu_nid_range *range)
                nm_range_insert(range,
                                &nm_range_tree->nmrt_range_interval_root);
        } else {
+               if (range_find(config, &range->rn_start, &range->rn_end,
+                              range->rn_netmask))
+                       return -EEXIST;
+
                list_add(&range->rn_collect, &config->nmc_netmask_setup);
        }
        return 0;
@@ -220,7 +281,7 @@ void range_delete(struct nodemap_config *config, struct lu_nid_range *range)
 }
 
 /*
- * search the interval tree for an nid within a range
+ * search the interval tree for a nid within a range
  *
  * \param      nid             nid to search for
  */
@@ -234,16 +295,15 @@ struct lu_nid_range *range_search(struct nodemap_config *config,
                return nm_range_iter_first(&nm_range_tree->nmrt_range_interval_root,
                                           lnet_nid_to_nid4(nid),
                                           lnet_nid_to_nid4(nid));
-       } else if (!list_empty(&config->nmc_netmask_setup)) {
+       }
+
+       if (!list_empty(&config->nmc_netmask_setup)) {
                struct lu_nid_range *range, *range_temp;
 
-               /* FIXME. We scan the config for the large NIDs. Each range
-                * only contains one large NID for now.
-                */
                list_for_each_entry_safe(range, range_temp,
                                         &config->nmc_netmask_setup,
                                         rn_collect) {
-                       if (nid_same(&range->rn_start, nid))
+                       if (cfs_match_nid(nid, &range->rn_nidlist))
                                return range;
                }
        }
index 81661a2..01177a8 100755 (executable)
@@ -1214,7 +1214,7 @@ create_fops_nodemaps() {
        for client in $clients; do
                local client_ip=$(host_nids_address $client $NETTYPE)
                local client_nid=$(h2nettype $client_ip)
-               [[ "$client_nid" =~ ":" ]] && client_nid+="/128"
+
                do_facet mgs $LCTL nodemap_add c${i} || return 1
                do_facet mgs $LCTL nodemap_add_range    \
                        --name c${i} --range $client_nid || {
@@ -5073,10 +5073,10 @@ test_55() {
 
        client_ip=$(host_nids_address $HOSTNAME $NETTYPE)
        client_nid=$(h2nettype $client_ip)
-       [[ "$client_nid" =~ ":" ]] && client_nid+="/128"
        do_facet mgs $LCTL nodemap_add c0
        do_facet mgs $LCTL nodemap_add_range \
-                --name c0 --range $client_nid
+                --name c0 --range $client_nid ||
+               error "Add range $client_nid to c0 failed rc = $?"
        do_facet mgs $LCTL nodemap_modify --name c0 \
                 --property admin --value 0
        do_facet mgs $LCTL nodemap_modify --name c0 \
@@ -5519,7 +5519,6 @@ setup_61() {
 
        client_ip=$(host_nids_address $HOSTNAME $NETTYPE)
        client_nid=$(h2nettype $client_ip)
-       [[ "$client_nid" =~ ":" ]] && client_nid+="/128"
        do_facet mgs $LCTL nodemap_add c0
        do_facet mgs $LCTL nodemap_add_range \
                 --name c0 --range $client_nid || {
@@ -5831,10 +5830,10 @@ setup_64() {
 
        client_ip=$(host_nids_address $HOSTNAME $NETTYPE)
        client_nid=$(h2nettype $client_ip)
-       [[ "$client_nid" =~ ":" ]] && client_nid+="/128"
        do_facet mgs $LCTL nodemap_add c0
        do_facet mgs $LCTL nodemap_add_range \
-                --name c0 --range $client_nid
+                --name c0 --range $client_nid ||
+               error "Add range $client_nid to c0 failed rc = $?"
        do_facet mgs $LCTL nodemap_modify --name c0 \
                 --property admin --value 1
        do_facet mgs $LCTL nodemap_modify --name c0 \
index d1599f3..92ab07b 100644 (file)
@@ -4219,25 +4219,11 @@ static int parse_nid_range(char *nodemap_range, char *nid_range, int range_len)
        char min_nid[LNET_NIDSTR_SIZE + 1];
        char max_nid[LNET_NIDSTR_SIZE + 1];
        struct list_head nidlist;
-       char *netmask = NULL;
        int rc = 0;
 
-       netmask = strchr(nodemap_range, '/');
-       if (netmask) {
-               unsigned long mask;
-
-               /* FIXME !!! Only 128 netmask is supported. This means
-                * nodemap will only support one large NID.
-                */
-               mask = strtoul(++netmask, NULL, 10);
-               if (mask < 0)
-                       return -errno;
-
-               if (mask != 128)
-                       return -ERANGE;
-
+       if (strchr(nodemap_range, '/') || strchr(nodemap_range, ':')) {
                strncpy(nid_range, nodemap_range, range_len);
-               return rc;
+               return 0;
        }
 
        INIT_LIST_HEAD(&nidlist);