LU-14391 utils: handle very large YAML data sets.

Some functionality for Lustre and even LNet can return huge
amounts of Netlink data that can overwhelm the internal libyaml
buffers. To resolve this we can create a resizable internal
buffer to collect all the Netlink data that is formated into
YAML. After the message has been completed we can feed this
data in chunk sizes the smaller internal libyaml library can
handle. The libyaml library internal buffer is a rolling buffer
so it will updated when we exceed its internal size. This will
allows collecting every single type of Lustre stat in one go
and for sites that have very large LNet router setups.

Test-Parameters: trivial
Change-Id: I20fdbb19b0f3de3ab52e8ad568c6926f61f627b9
Signed-off-by: James Simmons <jsimmons@infradead.org>
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54132
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Chris Horn <chris.horn@hpe.com>
Reviewed-by: Timothy Day <timday@amazon.com>
Reviewed-by: Frank Sehr <fsehr@whamcloud.com>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/utils/lnetconfig/liblnetconfig_netlink.c b/lnet/utils/lnetconfig/liblnetconfig_netlink.c
index 3c7e7f7..1f40471 100644 (file)
--- a/lnet/utils/lnetconfig/liblnetconfig_netlink.c
+++ b/lnet/utils/lnetconfig/liblnetconfig_netlink.c
@@ -425,7 +425,10 @@ struct yaml_nl_node {
 
 struct yaml_netlink_input {
        yaml_parser_t           *parser;
+       void                    *start;
+       void                    *read;
        void                    *buffer;
+       void                    *end;
        const char              *errmsg;
        int                     error;
        struct nl_sock          *nl;
@@ -1043,8 +1046,21 @@ static int yaml_netlink_msg_parse(struct nl_msg *msg, void *arg)
                if (lnet_genlmsg_parse(nlh, 0, attrs, maxtype, policy))
                        return NL_SKIP;
 
-               size = data->parser->raw_buffer.end -
-                      (unsigned char *)data->buffer;
+               size = data->end - data->buffer;
+               if (size < 1024) {
+                       size_t len = (data->end - data->start) * 2;
+                       size_t off = data->buffer - data->start;
+
+                       data->start = realloc(data->start, len);
+                       if (!data->start)
+                               return NL_STOP;
+                       data->end = data->start + len;
+
+                       data->buffer = data->start + off;
+                       data->read = data->start;
+
+                       size = data->end - data->buffer;
+               }
                yaml_parse_value_list(data, &size, attrs,
                                      &data->cur->keys.lkl_list[1]);
        }
@@ -1143,34 +1159,36 @@ static int yaml_netlink_read_handler(void *arg, unsigned char *buffer,
                                     size_t size, size_t *size_read)
 {
        struct yaml_netlink_input *data = arg;
-       struct nl_sock *nl = data->nl;
-       struct nl_cb *cb;
        int rc = 0;
 
-       if (data->complete) {
-               *size_read = 0;
-               return 1;
-       }
-
-       data->buffer = buffer;
-
-       cb = nl_socket_get_cb(nl);
-       rc = nl_recvmsgs_report(nl, cb);
-       if (rc == -NLE_INTR) {
-               *size_read = 0;
-               return 1;
-       } else if (!data->errmsg && rc < 0) {
-               data->errmsg = nl_geterror(rc);
-               return 0;
-       } else if (data->parser->error) {
-               /* data->errmsg is set in NL_CB_FINISH */
-               return 0;
+       /* First collect the Netlink data and then transfer it
+        * into the internal libyaml buffers.
+        */
+       if (!data->complete) {
+               struct nl_cb *cb = nl_socket_get_cb(data->nl);
+
+               rc = nl_recvmsgs_report(data->nl, cb);
+               if (rc == -NLE_INTR) {
+                       *size_read = 0;
+                       return 1;
+               } else if (!data->errmsg && rc < 0) {
+                       data->errmsg = nl_geterror(rc);
+                       return 0;
+               } else if (data->parser->error) {
+                       /* data->errmsg is set in NL_CB_FINISH */
+                       return 0;
+               }
        }
-
-       rc = (unsigned char *)data->buffer - buffer;
+       rc = data->buffer - data->read;
        if ((int)size > rc)
                size = rc;
 
+       if (size) {
+               memcpy(buffer, data->read, size);
+               data->read += size;
+       } else if (data->complete) {
+               free(data->start);
+       }
        *size_read = size;
        return 1;
 }
@@ -1229,6 +1247,10 @@ yaml_parser_set_input_netlink(yaml_parser_t *reply, struct nl_sock *nl,
                goto failed;
        }
 
+       buf->start = malloc(65536);
+       buf->end = buf->start + 65536;
+       buf->buffer = buf->start;
+       buf->read = buf->start;
        buf->nl = nl;
        buf->async = stream;
        buf->parser = reply;