From 0bb4325ba33560d6ddae8d09610bc22b76ff88bb Mon Sep 17 00:00:00 2001 From: Etienne AUJAMES Date: Thu, 7 Mar 2024 12:27:13 -0500 Subject: [PATCH] LU-12452 lnet: inherit default lnd tunings from modparams When a network is added dynamically (via Netlink), LNet assumes that all the "unset" or default LND parameters are 0. But for some use cases, 0 could be a valid value. This patch modifies the callback lnet_lnd.lnd_nl_set() to set default values: a NULL attr argument means "set the default value". Test-Parameters: trivial testlist=sanity-lnet Signed-off-by: Etienne AUJAMES Change-Id: Ifb91ae63d96131ed87d9fae7d91b8b18df4c9ce9 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54078 Tested-by: jenkins Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Serguei Smirnov Reviewed-by: Frank Sehr Reviewed-by: Oleg Drokin --- lnet/klnds/gnilnd/gnilnd.c | 3 +++ lnet/klnds/kfilnd/kfilnd.c | 3 +++ lnet/klnds/o2iblnd/o2iblnd.c | 48 ++++++++++++++++++++++++++++++++++ lnet/klnds/o2iblnd/o2iblnd.h | 1 + lnet/klnds/o2iblnd/o2iblnd_modparams.c | 22 ++++++++-------- lnet/klnds/socklnd/socklnd.c | 26 ++++++++++++++++++ lnet/klnds/socklnd/socklnd.h | 1 + lnet/klnds/socklnd/socklnd_modparams.c | 8 +++--- lnet/lnet/api-ni.c | 36 ++++++++++++++++++++----- 9 files changed, 127 insertions(+), 21 deletions(-) diff --git a/lnet/klnds/gnilnd/gnilnd.c b/lnet/klnds/gnilnd/gnilnd.c index e507c02..675c885 100644 --- a/lnet/klnds/gnilnd/gnilnd.c +++ b/lnet/klnds/gnilnd/gnilnd.c @@ -48,6 +48,9 @@ kgnilnd_nl_set(int cmd, struct nlattr *attr, int type, void *data) if (cmd != LNET_CMD_NETS) return -EOPNOTSUPP; + if (!attr) + return 0; + if (nla_type(attr) != LN_SCALAR_ATTR_INT_VALUE) return -EINVAL; diff --git a/lnet/klnds/kfilnd/kfilnd.c b/lnet/klnds/kfilnd/kfilnd.c index 4be2e9b..df57836 100644 --- a/lnet/klnds/kfilnd/kfilnd.c +++ b/lnet/klnds/kfilnd/kfilnd.c @@ -400,6 +400,9 @@ kfilnd_nl_set(int cmd, struct nlattr *attr, int type, void *data) if (cmd != LNET_CMD_NETS) return -EOPNOTSUPP; + if (!attr) + return 0; + lnd_kfi = &tunables->lnd_tun_u.lnd_kfi; switch (type) { diff --git a/lnet/klnds/o2iblnd/o2iblnd.c b/lnet/klnds/o2iblnd/o2iblnd.c index acdd22d..4277f8b 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.c +++ b/lnet/klnds/o2iblnd/o2iblnd.c @@ -1294,6 +1294,49 @@ kiblnd_nl_get(int cmd, struct sk_buff *msg, int type, void *data) return 0; } +static inline void +kiblnd_nl_set_default(int cmd, int type, void *data) +{ + struct lnet_lnd_tunables *tunables = data; + struct lnet_ioctl_config_o2iblnd_tunables *lt; + struct lnet_ioctl_config_o2iblnd_tunables *df; + + lt = &tunables->lnd_tun_u.lnd_o2ib; + df = &kib_default_tunables; + switch (type) { + case LNET_NET_O2IBLND_TUNABLES_ATTR_HIW_PEER_CREDITS: + lt->lnd_peercredits_hiw = df->lnd_peercredits_hiw; + break; + case LNET_NET_O2IBLND_TUNABLES_ATTR_MAP_ON_DEMAND: + lt->lnd_map_on_demand = df->lnd_map_on_demand; + break; + case LNET_NET_O2IBLND_TUNABLES_ATTR_CONCURRENT_SENDS: + lt->lnd_concurrent_sends = df->lnd_concurrent_sends; + break; + case LNET_NET_O2IBLND_TUNABLES_ATTR_FMR_POOL_SIZE: + lt->lnd_fmr_pool_size = df->lnd_fmr_pool_size; + break; + case LNET_NET_O2IBLND_TUNABLES_ATTR_FMR_FLUSH_TRIGGER: + lt->lnd_fmr_flush_trigger = df->lnd_fmr_flush_trigger; + break; + case LNET_NET_O2IBLND_TUNABLES_ATTR_FMR_CACHE: + lt->lnd_fmr_cache = df->lnd_fmr_cache; + break; + case LNET_NET_O2IBLND_TUNABLES_ATTR_NTX: + lt->lnd_ntx = df->lnd_ntx; + break; + case LNET_NET_O2IBLND_TUNABLES_ATTR_LND_TIMEOUT: + lt->lnd_timeout = df->lnd_timeout; + break; + case LNET_NET_O2IBLND_TUNABLES_ATTR_CONNS_PER_PEER: + lt->lnd_conns_per_peer = df->lnd_conns_per_peer; + fallthrough; + default: + break; + } + +} + static int kiblnd_nl_set(int cmd, struct nlattr *attr, int type, void *data) { @@ -1304,6 +1347,11 @@ kiblnd_nl_set(int cmd, struct nlattr *attr, int type, void *data) if (cmd != LNET_CMD_NETS) return -EOPNOTSUPP; + if (!attr) { + kiblnd_nl_set_default(cmd, type, data); + return 0; + } + if (nla_type(attr) != LN_SCALAR_ATTR_INT_VALUE) return -EINVAL; diff --git a/lnet/klnds/o2iblnd/o2iblnd.h b/lnet/klnds/o2iblnd/o2iblnd.h index 021fa5c..e7a28ab 100644 --- a/lnet/klnds/o2iblnd/o2iblnd.h +++ b/lnet/klnds/o2iblnd/o2iblnd.h @@ -140,6 +140,7 @@ struct kib_tunables { }; extern struct kib_tunables kiblnd_tunables; +extern struct lnet_ioctl_config_o2iblnd_tunables kib_default_tunables; #define IBLND_MSG_QUEUE_SIZE_V1 8 /* V1 only : # messages/RDMAs in-flight */ #define IBLND_CREDIT_HIGHWATER_V1 7 /* V1 only : when eagerly to return credits */ diff --git a/lnet/klnds/o2iblnd/o2iblnd_modparams.c b/lnet/klnds/o2iblnd/o2iblnd_modparams.c index 4e32eaa..711583c 100644 --- a/lnet/klnds/o2iblnd/o2iblnd_modparams.c +++ b/lnet/klnds/o2iblnd/o2iblnd_modparams.c @@ -201,7 +201,7 @@ struct kib_tunables kiblnd_tunables = { .kib_use_fastreg_gaps = &use_fastreg_gaps, }; -static struct lnet_ioctl_config_o2iblnd_tunables default_tunables; +struct lnet_ioctl_config_o2iblnd_tunables kib_default_tunables; /* # messages/RDMAs in-flight */ int @@ -227,7 +227,7 @@ kiblnd_tunables_setup(struct lnet_ni *ni) */ if (!ni->ni_lnd_tunables_set) memcpy(&ni->ni_lnd_tunables.lnd_tun_u.lnd_o2ib, - &default_tunables, sizeof(*tunables)); + &kib_default_tunables, sizeof(*tunables)); tunables = &ni->ni_lnd_tunables.lnd_tun_u.lnd_o2ib; @@ -325,14 +325,14 @@ kiblnd_tunables_setup(struct lnet_ni *ni) int kiblnd_tunables_init(void) { - default_tunables.lnd_version = CURRENT_LND_VERSION; - default_tunables.lnd_peercredits_hiw = peer_credits_hiw; - default_tunables.lnd_map_on_demand = map_on_demand; - default_tunables.lnd_concurrent_sends = concurrent_sends; - default_tunables.lnd_fmr_pool_size = fmr_pool_size; - default_tunables.lnd_fmr_flush_trigger = fmr_flush_trigger; - default_tunables.lnd_fmr_cache = fmr_cache; - default_tunables.lnd_ntx = ntx; - default_tunables.lnd_conns_per_peer = conns_per_peer; + kib_default_tunables.lnd_version = CURRENT_LND_VERSION; + kib_default_tunables.lnd_peercredits_hiw = peer_credits_hiw; + kib_default_tunables.lnd_map_on_demand = map_on_demand; + kib_default_tunables.lnd_concurrent_sends = concurrent_sends; + kib_default_tunables.lnd_fmr_pool_size = fmr_pool_size; + kib_default_tunables.lnd_fmr_flush_trigger = fmr_flush_trigger; + kib_default_tunables.lnd_fmr_cache = fmr_cache; + kib_default_tunables.lnd_ntx = ntx; + kib_default_tunables.lnd_conns_per_peer = conns_per_peer; return 0; } diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index 1947d36..2c2070c 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -837,6 +837,27 @@ ksocknal_nl_get(int cmd, struct sk_buff *msg, int type, void *data) return 0; } +static inline void +ksocknal_nl_set_default(int cmd, int type, void *data) +{ + struct lnet_lnd_tunables *tunables = data; + struct lnet_ioctl_config_socklnd_tunables *lt; + struct lnet_ioctl_config_socklnd_tunables *df; + + lt = &tunables->lnd_tun_u.lnd_sock; + df = &ksock_default_tunables; + switch (type) { + case LNET_NET_SOCKLND_TUNABLES_ATTR_CONNS_PER_PEER: + lt->lnd_conns_per_peer = df->lnd_conns_per_peer; + break; + case LNET_NET_SOCKLND_TUNABLES_ATTR_LND_TIMEOUT: + lt->lnd_timeout = df->lnd_timeout; + fallthrough; + default: + break; + } +} + static int ksocknal_nl_set(int cmd, struct nlattr *attr, int type, void *data) { @@ -847,6 +868,11 @@ ksocknal_nl_set(int cmd, struct nlattr *attr, int type, void *data) if (cmd != LNET_CMD_NETS) return -EOPNOTSUPP; + if (!attr) { + ksocknal_nl_set_default(cmd, type, data); + return 0; + } + if (nla_type(attr) != LN_SCALAR_ATTR_INT_VALUE) return -EINVAL; diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h index 7c6a18d..ea0f3e6 100644 --- a/lnet/klnds/socklnd/socklnd.h +++ b/lnet/klnds/socklnd/socklnd.h @@ -436,6 +436,7 @@ struct ksock_connreq { extern struct ksock_nal_data ksocknal_data; extern struct ksock_tunables ksocknal_tunables; +extern struct lnet_ioctl_config_socklnd_tunables ksock_default_tunables; #define SOCKNAL_MATCH_NO 0 /* TX can't match type of connection */ #define SOCKNAL_MATCH_YES 1 /* TX matches type of connection */ diff --git a/lnet/klnds/socklnd/socklnd_modparams.c b/lnet/klnds/socklnd/socklnd_modparams.c index bd97f1a..4fc1246 100644 --- a/lnet/klnds/socklnd/socklnd_modparams.c +++ b/lnet/klnds/socklnd/socklnd_modparams.c @@ -185,7 +185,7 @@ static inline bool is_native_host(void) } struct ksock_tunables ksocknal_tunables; -static struct lnet_ioctl_config_socklnd_tunables default_tunables; +struct lnet_ioctl_config_socklnd_tunables ksock_default_tunables; #ifdef HAVE_ETHTOOL_LINK_SETTINGS static int ksocklnd_ni_get_eth_intf_speed(struct lnet_ni *ni) @@ -270,8 +270,8 @@ static int ksocklnd_lookup_conns_per_peer(struct lnet_ni *ni) int ksocknal_tunables_init(void) { - default_tunables.lnd_version = CURRENT_LND_VERSION; - default_tunables.lnd_conns_per_peer = conns_per_peer; + ksock_default_tunables.lnd_version = CURRENT_LND_VERSION; + ksock_default_tunables.lnd_conns_per_peer = conns_per_peer; /* initialize ksocknal_tunables structure */ ksocknal_tunables.ksnd_timeout = &sock_timeout; @@ -344,7 +344,7 @@ void ksocknal_tunables_setup(struct lnet_ni *ni) /* If no tunables specified, setup default tunables */ if (!ni->ni_lnd_tunables_set) memcpy(&ni->ni_lnd_tunables.lnd_tun_u.lnd_sock, - &default_tunables, sizeof(*tunables)); + &ksock_default_tunables, sizeof(*tunables)); tunables = &ni->ni_lnd_tunables.lnd_tun_u.lnd_sock; diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index b8c2edc..1fe475f 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -5921,6 +5921,26 @@ static int lnet_genl_parse_lnd_tunables(struct nlattr *settings, return rc; } +static inline void +lnet_genl_init_tunables(const struct lnet_lnd *lnd, + struct lnet_ioctl_config_lnd_tunables *tun) +{ + const struct ln_key_list *list = lnd ? lnd->lnd_keys : NULL; + int i; + + tun->lt_cmn.lct_peer_timeout = -1; + tun->lt_cmn.lct_peer_tx_credits = -1; + tun->lt_cmn.lct_peer_rtr_credits = -1; + tun->lt_cmn.lct_max_tx_credits = -1; + + if (!list || !lnd->lnd_nl_set || !list->lkl_maxattr) + return; + + /* init lnd tunables with default values */ + for (i = 1; i <= list->lkl_maxattr; i++) + lnd->lnd_nl_set(LNET_CMD_NETS, NULL, i, &tun->lt_tun); +} + static int lnet_genl_parse_local_ni(struct nlattr *entry, struct genl_info *info, int net_id, struct lnet_ioctl_config_ni *conf, @@ -5928,10 +5948,19 @@ lnet_genl_parse_local_ni(struct nlattr *entry, struct genl_info *info, { struct lnet_ioctl_config_lnd_tunables *tun; struct lnet_nid nid = LNET_ANY_NID; + const struct lnet_lnd *lnd = NULL; struct nlattr *settings; int healthv = -1; int rem3, rc = 0; + if (net_id != LNET_NET_ANY) { + lnd = lnet_load_lnd(LNET_NETTYP(net_id)); + if (IS_ERR(lnd)) { + GENL_SET_ERR_MSG(info, "LND type not supported"); + RETURN(PTR_ERR(lnd)); + } + } + LIBCFS_ALLOC(tun, sizeof(struct lnet_ioctl_config_lnd_tunables)); if (!tun) { GENL_SET_ERR_MSG(info, "cannot allocate memory for tunables"); @@ -5939,10 +5968,7 @@ lnet_genl_parse_local_ni(struct nlattr *entry, struct genl_info *info, } /* Use LND defaults */ - tun->lt_cmn.lct_peer_timeout = -1; - tun->lt_cmn.lct_peer_tx_credits = -1; - tun->lt_cmn.lct_peer_rtr_credits = -1; - tun->lt_cmn.lct_max_tx_credits = -1; + lnet_genl_init_tunables(lnd, tun); conf->lic_ncpts = 0; nla_for_each_nested(settings, entry, rem3) { @@ -6050,8 +6076,6 @@ lnet_genl_parse_local_ni(struct nlattr *entry, struct genl_info *info, GOTO(out, rc); } } else if ((nla_strcmp(settings, "lnd tunables") == 0)) { - const struct lnet_lnd *lnd; - settings = nla_next(settings, &rem3); if (nla_type(settings) != LN_SCALAR_ATTR_LIST) { -- 1.8.3.1