From d26f2a53b66efdb0dfe374da5e25a99489b55add Mon Sep 17 00:00:00 2001 From: James Simmons Date: Fri, 26 Apr 2024 13:15:02 -0400 Subject: [PATCH] LU-14391 lnet: optimize the Netlink packet size for routes Currently Netlink by default sets its maximum packet size to send back to user land to 64K. Some sites setup many routes, above ~430, which exceed this limit. We can avoid this limitation by calculate about the actually size of the netlink packet and setting cb->min_dump_alloc. The new max is then 4GB which should be plenty (27K of routes) Test-Parameters: trivial testlist=sanity-lnet Change-Id: Ica01f0cf290992a5d27b8ac2d09508d0a6e8151a Signed-off-by: James Simmons Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54844 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Serguei Smirnov Reviewed-by: Frank Sehr Reviewed-by: Oleg Drokin --- lnet/lnet/api-ni.c | 24 ++++++++++++++++++++++++ lustre/tests/sanity-lnet.sh | 15 +++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index e2bb7c7..4c801e9 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -6789,6 +6789,21 @@ failed_alloc: return rc; } +/* Size of the message send by lnet_genl_send_scalar_list(). + * Length is from genlmsg_len() on the msg created. + */ +#define ROUTER_MSG_MIN_SIZE 284 +/* For 'value' packet it contains + * net LNET_NIDSTR_SIZE + * gateway LNET_NIDSTR_SIZE + * hop u32 + * priority u32 + * health sensit.. u32 + * state "down" largest string (5) + * type "single-hop" largest string (10) + */ +#define ROUTER_MSG_VALUES_SIZE (LNET_NIDSTR_SIZE * 2 + 27) + /* LNet route ->start() handler for GET requests */ static int lnet_route_show_start(struct netlink_callback *cb) { @@ -6796,6 +6811,7 @@ static int lnet_route_show_start(struct netlink_callback *cb) #ifdef HAVE_NL_PARSE_WITH_EXT_ACK struct netlink_ext_ack *extack = NULL; #endif + unsigned long len = ROUTER_MSG_MIN_SIZE; struct lnet_genl_route_list *rlist; int msg_len = genlmsg_len(gnlh); int rc = 0; @@ -6936,6 +6952,14 @@ static int lnet_route_show_start(struct netlink_callback *cb) } } } + + len += ROUTER_MSG_VALUES_SIZE * rlist->lgrl_count; + if (len > BIT(sizeof(cb->min_dump_alloc) << 3)) { + NL_SET_ERR_MSG(extack, "Netlink msg is too large"); + rc = -EMSGSIZE; + } else { + cb->min_dump_alloc = len; + } report_err: mutex_unlock(&the_lnet.ln_api_mutex); diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh index af84e49..d35e310 100755 --- a/lustre/tests/sanity-lnet.sh +++ b/lustre/tests/sanity-lnet.sh @@ -1708,6 +1708,21 @@ test_110() { } run_test 110 "Add NI using a specific TCP / IP address" +test_111() { + [[ $(uname -r | grep "3.10") ]] && + skip "Unsupported on RHEL7" + + reinit_dlc || return $? + add_net "${NETTYPE}" "${INTERFACES[0]}" + + for index in {2..500} + do + do_lnetctl route add --net ${NETTYPE}${index} --gateway ${GW_NID} + done + do_lnetctl route show || return $? +} +run_test 111 "Test many routes" + test_200() { [[ ${NETTYPE} == tcp* ]] || skip "Need tcp NETTYPE" -- 1.8.3.1