Whamcloud - gitweb
LU-10003 lnet: migrate old route API to Netlink / YAML API 40/50440/22
authorJames Simmons <jsimmons@infradead.org>
Sun, 9 Jul 2023 13:55:39 +0000 (09:55 -0400)
committerOleg Drokin <green@whamcloud.com>
Fri, 3 Nov 2023 04:01:30 +0000 (04:01 +0000)
With LNet route management now supported with the Netlink / YAML
API we can move the pre-MR route handling away from the
old ioctls. This allows use of large NIDs as well for IPv6
support.

Test-Parameters: trivial
Test-Parameters: testlist=conf-sanity env=ONLY="67"
Signed-off-by: James Simmons <jsimmons@infradead.org>
Change-Id: I1526d3e27967a10f556b8e31f15803659c0164bd
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/50440
Reviewed-by: Cyril Bordage <cbordage@whamcloud.com>
Reviewed-by: Chris Horn <chris.horn@hpe.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
lnet/lnet/api-ni.c
lustre/utils/portals.c

index c41ca00..6ae5bdf 100644 (file)
@@ -4500,16 +4500,14 @@ LNetCtl(unsigned int cmd, void *arg)
        }
 
        case IOC_LIBCFS_NOTIFY_ROUTER: {
-               time64_t deadline = ktime_get_real_seconds() - data->ioc_u64[0];
-
-               /* The deadline passed in by the user should be some time in
-                * seconds in the future since the UNIX epoch. We have to map
-                * that deadline to the wall clock.
+               /* Convert the user-supplied real time to monotonic.
+                * NB: "when" is always in the past
                 */
-               deadline += ktime_get_seconds();
+               time64_t when = ktime_get_seconds() -
+                               (ktime_get_real_seconds() - data->ioc_u64[0]);
+
                lnet_nid4_to_nid(data->ioc_nid, &nid);
-               return lnet_notify(NULL, &nid, data->ioc_flags, false,
-                                  deadline);
+               return lnet_notify(NULL, &nid, data->ioc_flags, false, when);
        }
 
        case IOC_LIBCFS_LNET_DIST:
@@ -7249,6 +7247,8 @@ static int lnet_route_cmd(struct sk_buff *skb, struct genl_info *info)
                u32 priority = 0, sensitivity = 1;
                struct lnet_nid gw_nid = LNET_ANY_NID;
                struct nlattr *route_prop;
+               bool alive = true;
+               s64 when = 0;
                int rem2;
 
                if (nla_type(attr) != LN_SCALAR_ATTR_LIST)
@@ -7318,6 +7318,39 @@ static int lnet_route_cmd(struct sk_buff *skb, struct genl_info *info)
                                                         "cannot parse gateway");
                                        GOTO(report_err, rc = -ENODEV);
                                }
+                       } else if (nla_strcmp(route_prop, "state") == 0) {
+                               route_prop = nla_next(route_prop, &rem2);
+                               if (nla_type(route_prop) !=
+                                   LN_SCALAR_ATTR_VALUE) {
+                                       GENL_SET_ERR_MSG(info,
+                                                        "state is invalid key");
+                                       GOTO(report_err, rc = -EINVAL);
+                               }
+
+                               if (nla_strcmp(route_prop, "down") == 0) {
+                                       alive = false;
+                               } else if (nla_strcmp(route_prop, "up") == 0) {
+                                       alive = true;
+                               } else {
+                                       GENL_SET_ERR_MSG(info,
+                                                        "status string bad value");
+                                       GOTO(report_err, rc = -EINVAL);
+                               }
+                       } else if (nla_strcmp(route_prop, "notify_time") == 0) {
+                               route_prop = nla_next(route_prop, &rem2);
+                               if (nla_type(route_prop) !=
+                                   LN_SCALAR_ATTR_INT_VALUE) {
+                                       GENL_SET_ERR_MSG(info,
+                                                        "notify_time is invalid key");
+                                       GOTO(report_err, rc = -EINVAL);
+                               }
+
+                               when = nla_get_s64(route_prop);
+                               if (ktime_get_real_seconds() < when) {
+                                       GENL_SET_ERR_MSG(info,
+                                                        "notify_time is in the future");
+                                       GOTO(report_err, rc = -EINVAL);
+                               }
                        } else if (nla_strcmp(route_prop, "hop") == 0) {
                                route_prop = nla_next(route_prop, &rem2);
                                if (nla_type(route_prop) !=
@@ -7381,7 +7414,21 @@ static int lnet_route_cmd(struct sk_buff *skb, struct genl_info *info)
                        GOTO(report_err, rc = -ENODEV);
                }
 
-               if (info->nlhdr->nlmsg_flags & NLM_F_CREATE) {
+               if (info->nlhdr->nlmsg_flags & NLM_F_REPLACE) {
+                       /* Convert the user-supplied real time to monotonic.
+                        * NB: "when" is always in the past
+                        */
+                       when = ktime_get_seconds() -
+                               (ktime_get_real_seconds() - when);
+
+                       mutex_unlock(&the_lnet.ln_api_mutex);
+                       rc = lnet_notify(NULL, &gw_nid, alive, false, when);
+                       mutex_lock(&the_lnet.ln_api_mutex);
+                       if (rc < 0)
+                               GOTO(report_err, rc);
+                       else if (the_lnet.ln_state != LNET_STATE_RUNNING)
+                               GOTO(report_err, rc = -ENETDOWN);
+               } else if (info->nlhdr->nlmsg_flags & NLM_F_CREATE) {
                        rc = lnet_add_route(net_id, hops, &gw_nid, priority,
                                            sensitivity);
                        if (rc < 0) {
index aac8b78..6be4c15 100644 (file)
@@ -1442,6 +1442,340 @@ jt_ptl_fail_nid(int argc, char **argv)
        return 0;
 }
 
+static int ptl_yaml_route_display(yaml_parser_t *reply)
+{
+       char gw[LNET_MAX_STR_LEN], net[18];
+       bool done = false, alive = false;
+       int hops = -1, prio = -1;
+       yaml_event_t event;
+       int rc;
+
+       /* Now parse the reply results */
+       while (!done) {
+               char *value;
+
+               rc = yaml_parser_parse(reply, &event);
+               if (rc == 0)
+                       break;
+
+               if (event.type == YAML_SEQUENCE_END_EVENT) {
+                       printf("net %18s hops %d gw %32.128s %s pri %u\n",
+                              net, hops, gw, alive ? "up" : "down",
+                              prio);
+                       memset(net, '\0', sizeof(net));
+                       memset(gw, '\0', sizeof(gw));
+                       prio = -1;
+                       hops = -1;
+               }
+
+               if (event.type != YAML_SCALAR_EVENT)
+                       goto skip;
+
+               value = (char *)event.data.scalar.value;
+               if (strcmp(value, "net") == 0) {
+                       yaml_event_delete(&event);
+                       rc = yaml_parser_parse(reply, &event);
+                       if (rc == 0) {
+                               yaml_event_delete(&event);
+                               goto free_reply;
+                       }
+
+                       value = (char *)event.data.scalar.value;
+                       strncpy(net, value, sizeof(net) - 1);
+               } else if (strcmp(value, "gateway") == 0) {
+                       yaml_event_delete(&event);
+                       rc = yaml_parser_parse(reply, &event);
+                       if (rc == 0) {
+                               yaml_event_delete(&event);
+                               goto free_reply;
+                       }
+
+                       value = (char *)event.data.scalar.value;
+                       strncpy(gw, value, sizeof(gw) - 1);
+               } else if (strcmp(value, "state") == 0) {
+                       yaml_event_delete(&event);
+                       rc = yaml_parser_parse(reply, &event);
+                       if (rc == 0) {
+                               yaml_event_delete(&event);
+                               goto free_reply;
+                       }
+
+                       value = (char *)event.data.scalar.value;
+                       if (strcmp(value, "up") == 0) {
+                               alive = true;
+                       } else if (strcmp(value, "down") == 0) {
+                               alive = false;
+                       }
+               } else if (strcmp(value, "hop") == 0) {
+                       yaml_event_delete(&event);
+                       rc = yaml_parser_parse(reply, &event);
+                       if (rc == 0) {
+                               yaml_event_delete(&event);
+                               goto free_reply;
+                       }
+
+                       value = (char *)event.data.scalar.value;
+                       hops = strtol(value, NULL, 10);
+               } else if (strcmp(value, "priority") == 0) {
+                       yaml_event_delete(&event);
+                       rc = yaml_parser_parse(reply, &event);
+                       if (rc == 0) {
+                               yaml_event_delete(&event);
+                               goto free_reply;
+                       }
+
+                       value = (char *)event.data.scalar.value;
+                       prio = strtol(value, NULL, 10);
+               }
+skip:
+               done = (event.type == YAML_STREAM_END_EVENT);
+               yaml_event_delete(&event);
+       }
+
+free_reply:
+       return rc;
+}
+
+static int ptl_yaml_route(char *nw, char *gws, int hops, int prio, bool enable,
+                         time_t notify_time, int flags, int version)
+{
+       struct nl_sock *sk = NULL;
+       const char *msg = NULL;
+       yaml_emitter_t output;
+       yaml_parser_t reply;
+       yaml_event_t event;
+       int rc;
+
+       sk = nl_socket_alloc();
+       if (!sk)
+               return -EOPNOTSUPP;
+
+       /* Setup parser to receive Netlink packets */
+       rc = yaml_parser_initialize(&reply);
+       if (rc == 0) {
+               nl_socket_free(sk);
+               return -EOPNOTSUPP;
+       }
+
+       rc = yaml_parser_set_input_netlink(&reply, sk, false);
+       if (rc == 0) {
+               msg = yaml_parser_get_reader_error(&reply);
+               goto free_reply;
+       }
+
+       /* Create Netlink emitter to send request to kernel */
+       rc = yaml_emitter_initialize(&output);
+       if (rc == 0) {
+               msg = "failed to initialize emitter";
+               goto free_reply;
+       }
+
+       rc = yaml_emitter_set_output_netlink(&output, sk, LNET_GENL_NAME,
+                                            version, LNET_CMD_ROUTES, flags);
+       if (rc == 0)
+               goto emitter_error;
+
+       yaml_emitter_open(&output);
+       yaml_document_start_event_initialize(&event, NULL, NULL, NULL, 0);
+       rc = yaml_emitter_emit(&output, &event);
+       if (rc == 0)
+               goto emitter_error;
+
+       yaml_mapping_start_event_initialize(&event, NULL,
+                                           (yaml_char_t *)YAML_MAP_TAG,
+                                           1, YAML_ANY_MAPPING_STYLE);
+       rc = yaml_emitter_emit(&output, &event);
+       if (rc == 0)
+               goto emitter_error;
+
+       yaml_scalar_event_initialize(&event, NULL,
+                                    (yaml_char_t *)YAML_STR_TAG,
+                                    (yaml_char_t *)"route",
+                                    strlen("route"), 1, 0,
+                                    YAML_PLAIN_SCALAR_STYLE);
+       rc = yaml_emitter_emit(&output, &event);
+       if (rc == 0)
+               goto emitter_error;
+
+       if (nw || gws) {
+               yaml_sequence_start_event_initialize(&event, NULL,
+                                                    (yaml_char_t *)YAML_SEQ_TAG,
+                                                    1,
+                                                    YAML_BLOCK_SEQUENCE_STYLE);
+               rc = yaml_emitter_emit(&output, &event);
+               if (rc == 0)
+                       goto emitter_error;
+
+               yaml_mapping_start_event_initialize(&event, NULL,
+                                                   (yaml_char_t *)YAML_MAP_TAG, 1,
+                                                   YAML_BLOCK_MAPPING_STYLE);
+               rc = yaml_emitter_emit(&output, &event);
+               if (rc == 0)
+                       goto emitter_error;
+
+               if (nw) {
+                       yaml_scalar_event_initialize(&event, NULL,
+                                                    (yaml_char_t *)YAML_STR_TAG,
+                                                    (yaml_char_t *)"net",
+                                                    strlen("net"), 1, 0,
+                                                    YAML_PLAIN_SCALAR_STYLE);
+                       rc = yaml_emitter_emit(&output, &event);
+                       if (rc == 0)
+                               goto emitter_error;
+
+                       yaml_scalar_event_initialize(&event, NULL,
+                                                    (yaml_char_t *)YAML_STR_TAG,
+                                                    (yaml_char_t *)nw,
+                                                    strlen(nw), 1, 0,
+                                                    YAML_PLAIN_SCALAR_STYLE);
+                       rc = yaml_emitter_emit(&output, &event);
+                       if (rc == 0)
+                               goto emitter_error;
+               }
+
+               if (gws) {
+                       yaml_scalar_event_initialize(&event, NULL,
+                                                    (yaml_char_t *)YAML_STR_TAG,
+                                                    (yaml_char_t *)"gateway",
+                                                    strlen("gateway"), 1, 0,
+                                                    YAML_PLAIN_SCALAR_STYLE);
+                       rc = yaml_emitter_emit(&output, &event);
+                       if (rc == 0)
+                               goto emitter_error;
+
+                       yaml_scalar_event_initialize(&event, NULL,
+                                                    (yaml_char_t *)YAML_STR_TAG,
+                                                    (yaml_char_t *)gws,
+                                                    strlen(gws), 1, 0,
+                                                    YAML_PLAIN_SCALAR_STYLE);
+                       rc = yaml_emitter_emit(&output, &event);
+                       if (rc == 0)
+                               goto emitter_error;
+               }
+
+               if (notify_time) {
+                       char when[INT_STRING_LEN];
+
+                       yaml_scalar_event_initialize(&event, NULL,
+                                                    (yaml_char_t *)YAML_STR_TAG,
+                                                    (yaml_char_t *)"notify_time",
+                                                    strlen("notify_time"), 1, 0,
+                                                    YAML_PLAIN_SCALAR_STYLE);
+                       rc = yaml_emitter_emit(&output, &event);
+                       if (rc == 0)
+                               goto emitter_error;
+
+                       snprintf(when, sizeof(when), "%ld", notify_time);
+                       yaml_scalar_event_initialize(&event, NULL,
+                                                    (yaml_char_t *)YAML_INT_TAG,
+                                                    (yaml_char_t *)when,
+                                                    strlen(when), 1, 0,
+                                                    YAML_PLAIN_SCALAR_STYLE);
+                       rc = yaml_emitter_emit(&output, &event);
+                       if (rc == 0)
+                               goto emitter_error;
+               }
+
+               yaml_scalar_event_initialize(&event, NULL,
+                                            (yaml_char_t *)YAML_STR_TAG,
+                                            (yaml_char_t *)"state",
+                                            strlen("state"), 1, 0,
+                                            YAML_PLAIN_SCALAR_STYLE);
+               rc = yaml_emitter_emit(&output, &event);
+               if (rc == 0)
+                       goto emitter_error;
+
+               if (enable)
+                       yaml_scalar_event_initialize(&event, NULL,
+                                                    (yaml_char_t *)YAML_STR_TAG,
+                                                    (yaml_char_t *)"up",
+                                                    strlen("up"), 1, 0,
+                                                    YAML_PLAIN_SCALAR_STYLE);
+               else
+                       yaml_scalar_event_initialize(&event, NULL,
+                                                    (yaml_char_t *)YAML_STR_TAG,
+                                                    (yaml_char_t *)"down",
+                                                    strlen("down"), 1, 0,
+                                                    YAML_PLAIN_SCALAR_STYLE);
+
+               rc = yaml_emitter_emit(&output, &event);
+               if (rc == 0)
+                       goto emitter_error;
+
+               yaml_mapping_end_event_initialize(&event);
+               rc = yaml_emitter_emit(&output, &event);
+               if (rc == 0)
+                       goto emitter_error;
+
+               yaml_sequence_end_event_initialize(&event);
+               rc = yaml_emitter_emit(&output, &event);
+               if (rc == 0)
+                       goto emitter_error;
+       } else {
+               yaml_scalar_event_initialize(&event, NULL,
+                                            (yaml_char_t *)YAML_STR_TAG,
+                                            (yaml_char_t *)"",
+                                            strlen(""), 1, 0,
+                                            YAML_PLAIN_SCALAR_STYLE);
+               rc = yaml_emitter_emit(&output, &event);
+               if (rc == 0)
+                       goto emitter_error;
+       }
+
+       yaml_mapping_end_event_initialize(&event);
+       rc = yaml_emitter_emit(&output, &event);
+       if (rc == 0)
+               goto emitter_error;
+
+       yaml_document_end_event_initialize(&event, 0);
+       rc = yaml_emitter_emit(&output, &event);
+       if (rc == 0)
+               goto emitter_error;
+
+       rc = yaml_emitter_close(&output);
+emitter_error:
+       if (rc == 0) {
+               yaml_emitter_log_error(&output, stderr);
+               rc = -EINVAL;
+       } else {
+               if (flags != NLM_F_DUMP) {
+                       yaml_document_t errmsg;
+
+                       rc = yaml_parser_load(&reply, &errmsg);
+                       if (rc == 1) {
+                               yaml_emitter_t debug;
+
+                               rc = yaml_emitter_initialize(&debug);
+                               if (rc == 1) {
+                                       yaml_emitter_set_indent(&debug,
+                                                               LNET_DEFAULT_INDENT);
+                                       yaml_emitter_set_output_file(&debug,
+                                                                    stdout);
+                                       rc = yaml_emitter_dump(&debug,
+                                                              &errmsg);
+                               } else if (rc == 0) {
+                                       yaml_emitter_log_error(&debug, stderr);
+                                       rc = -EINVAL;
+                               }
+                               yaml_emitter_delete(&debug);
+                       }
+                       yaml_document_delete(&errmsg);
+               } else {
+                       rc = ptl_yaml_route_display(&reply);
+               }
+               if (rc == 0)
+                       msg = yaml_parser_get_reader_error(&reply);
+       }
+       yaml_emitter_delete(&output);
+free_reply:
+       if (msg)
+               fprintf(stdout, "%s\n", msg);
+       yaml_parser_delete(&reply);
+       nl_socket_free(sk);
+
+       return rc == 1 ? 0 : rc;
+}
+
 int
 jt_ptl_add_route(int argc, char **argv)
 {
@@ -1486,6 +1820,14 @@ jt_ptl_add_route(int argc, char **argv)
                }
        }
 
+       rc = ptl_yaml_route(libcfs_net2str(g_net), argv[1], hops,
+                           priority, false, 0, NLM_F_CREATE, LNET_GENL_VERSION);
+       if (rc <= 0) {
+               if (rc == -EOPNOTSUPP)
+                       goto old_api;
+               return rc;
+       }
+old_api:
        LIBCFS_IOC_INIT_V2(data, cfg_hdr);
        data.cfg_net = g_net;
        data.cfg_config_u.cfg_route.rtr_hop = hops;
@@ -1519,6 +1861,14 @@ jt_ptl_del_route(int argc, char **argv)
                return -1;
        }
 
+       rc = ptl_yaml_route(g_net_set ? libcfs_net2str(g_net) : NULL, argv[1],
+                           -1, -1, false, 0, 0, LNET_GENL_VERSION);
+       if (rc <= 0) {
+               if (rc == -EOPNOTSUPP)
+                       goto old_api;
+               return rc;
+       }
+old_api:
        LIBCFS_IOC_INIT_V2(data, cfg_hdr);
        data.cfg_net = g_net_set ? g_net : LNET_NET_ANY;
        data.cfg_nid = nid;
@@ -1575,6 +1925,14 @@ jt_ptl_notify_router(int argc, char **argv)
                return -1;
        }
 
+       rc = ptl_yaml_route(g_net_set ? libcfs_net2str(g_net) : NULL, argv[1],
+                           -1, -1, enable, when, NLM_F_REPLACE, LNET_GENL_VERSION);
+       if (rc <= 0) {
+               if (rc == -EOPNOTSUPP)
+                       goto old_api;
+               return rc;
+       }
+old_api:
        LIBCFS_IOC_INIT(data);
        data.ioc_nid = nid;
        data.ioc_flags = enable;
@@ -1599,10 +1957,18 @@ jt_ptl_print_routes(int argc, char **argv)
        int index;
        __u32 net;
        lnet_nid_t nid;
-       unsigned int hops;
+       int hops;
        int alive;
        unsigned int pri;
 
+       rc = ptl_yaml_route(NULL, NULL, -1, -1, false, 0, NLM_F_DUMP,
+                           LNET_GENL_VERSION);
+       if (rc <= 0) {
+               if (rc == -EOPNOTSUPP)
+                       goto old_api;
+               return rc;
+       }
+old_api:
        for (index = 0; ; index++) {
                LIBCFS_IOC_INIT_V2(data, cfg_hdr);
                data.cfg_count = index;
@@ -1617,7 +1983,7 @@ jt_ptl_print_routes(int argc, char **argv)
                alive   = data.cfg_config_u.cfg_route.rtr_flags & LNET_RT_ALIVE;
                pri     = data.cfg_config_u.cfg_route.rtr_priority;
 
-               printf("net %18s hops %u gw %32s %s pri %u\n",
+               printf("net %18s hops %d gw %32s %s pri %u\n",
                       libcfs_net2str(net), hops,
                       libcfs_nid2str(nid), alive ? "up" : "down", pri);
        }