From a5cbe7883db6d77b82fbd83ad4c662499421d229 Mon Sep 17 00:00:00 2001 From: Serguei Smirnov Date: Mon, 2 Aug 2021 10:48:35 -0400 Subject: [PATCH] LU-12815 socklnd: allow dynamic setting of conns_per_peer Modify lnetctl and associated code to allow dynamic setting of conns_per_peer lnd parameter per ni. The parameter can be set for a specific active nid: lnetctl net set --nid 192.168.122.10@tcp --conns-per-peer=4 Or when adding a new net, taking effect on the new nid: lnetctl net add --net tcp --if eth0 --conns-per-peer=1 By default, conns_per_peer value specified as the module parameter shall be used. Test-Parameters: trivial testlist=sanity-lnet Signed-off-by: Serguei Smirnov Change-Id: I11625b9ad61f0311c294001a38b7855465491aaf Reviewed-on: https://review.whamcloud.com/41463 Tested-by: jenkins Reviewed-by: Andreas Dilger Reviewed-by: James Simmons Tested-by: Maloo Reviewed-by: Oleg Drokin --- lnet/include/lnet/lib-lnet.h | 3 + lnet/include/uapi/linux/lnet/libcfs_ioctl.h | 3 +- lnet/include/uapi/linux/lnet/lnet-dlc.h | 14 +++ lnet/klnds/socklnd/socklnd.c | 70 +++++++------- lnet/klnds/socklnd/socklnd.h | 5 +- lnet/klnds/socklnd/socklnd_modparams.c | 55 ++++++++++- lnet/lnet/api-ni.c | 44 ++++++++- lnet/utils/lnetconfig/liblnetconfig.c | 75 +++++++++++++-- lnet/utils/lnetconfig/liblnetconfig.h | 18 +++- lnet/utils/lnetconfig/liblnetconfig_lnd.c | 36 ++++++- lnet/utils/lnetctl.c | 140 +++++++++++++++++++++------- lustre/tests/sanity-lnet.sh | 41 ++++++++ 12 files changed, 418 insertions(+), 86 deletions(-) diff --git a/lnet/include/lnet/lib-lnet.h b/lnet/include/lnet/lib-lnet.h index 47bae48..0e53550 100644 --- a/lnet/include/lnet/lib-lnet.h +++ b/lnet/include/lnet/lib-lnet.h @@ -78,6 +78,9 @@ extern struct lnet the_lnet; /* THE network */ #define DEFAULT_PEER_CREDITS 8 #define DEFAULT_CREDITS 256 +/* default number of connections per peer */ +#define DEFAULT_CONNS_PER_PEER 1 + #ifdef HAVE_KERN_SOCK_GETNAME_2ARGS #define lnet_kernel_getpeername(sock, addr, addrlen) \ kernel_getpeername(sock, addr) diff --git a/lnet/include/uapi/linux/lnet/libcfs_ioctl.h b/lnet/include/uapi/linux/lnet/libcfs_ioctl.h index a17eae5..1bcf47b 100644 --- a/lnet/include/uapi/linux/lnet/libcfs_ioctl.h +++ b/lnet/include/uapi/linux/lnet/libcfs_ioctl.h @@ -154,7 +154,8 @@ struct libcfs_ioctl_data { #define IOC_LIBCFS_GET_UDSP _IOWR(IOC_LIBCFS_TYPE, 108, IOCTL_CONFIG_SIZE) #define IOC_LIBCFS_GET_CONST_UDSP_INFO _IOWR(IOC_LIBCFS_TYPE, 109, IOCTL_CONFIG_SIZE) #define IOC_LIBCFS_RESET_LNET_STATS _IOWR(IOC_LIBCFS_TYPE, 110, IOCTL_CONFIG_SIZE) -#define IOC_LIBCFS_MAX_NR 110 +#define IOC_LIBCFS_SET_CONNS_PER_PEER _IOWR(IOC_LIBCFS_TYPE, 111, IOCTL_CONFIG_SIZE) +#define IOC_LIBCFS_MAX_NR 111 extern int libcfs_ioctl_data_adjust(struct libcfs_ioctl_data *data); diff --git a/lnet/include/uapi/linux/lnet/lnet-dlc.h b/lnet/include/uapi/linux/lnet/lnet-dlc.h index 0ed61bd..fccadbe 100644 --- a/lnet/include/uapi/linux/lnet/lnet-dlc.h +++ b/lnet/include/uapi/linux/lnet/lnet-dlc.h @@ -81,9 +81,16 @@ struct lnet_ioctl_config_o2iblnd_tunables { __u16 lnd_ntx; }; +struct lnet_ioctl_config_socklnd_tunables { + __u32 lnd_version; + __u16 lnd_conns_per_peer; + __u16 lnd_pad; +}; + struct lnet_lnd_tunables { union { struct lnet_ioctl_config_o2iblnd_tunables lnd_o2ib; + struct lnet_ioctl_config_socklnd_tunables lnd_sock; } lnd_tun_u; }; @@ -280,6 +287,13 @@ struct lnet_ioctl_reset_health_cfg { lnet_nid_t rh_nid; }; +struct lnet_ioctl_reset_conns_per_peer_cfg { + struct libcfs_ioctl_hdr rcpp_hdr; + __u16 rcpp_all:1; + __s16 rcpp_value; + lnet_nid_t rcpp_nid; +}; + struct lnet_ioctl_recovery_list { struct libcfs_ioctl_hdr rlst_hdr; enum lnet_health_type rlst_type:32; diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index fef2edf..ab4adb4 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -36,6 +36,7 @@ * Author: Eric Barton */ +#include #include #include "socklnd.h" #include @@ -134,6 +135,7 @@ ksocknal_create_conn_cb(struct sockaddr *addr) conn_cb->ksnr_ctrl_conn_count = 0; conn_cb->ksnr_blki_conn_count = 0; conn_cb->ksnr_blko_conn_count = 0; + conn_cb->ksnr_max_conns = 0; return conn_cb; } @@ -393,6 +395,19 @@ ksocknal_get_conn_count_by_type(struct ksock_conn_cb *conn_cb, return count; } +static unsigned int +ksocknal_get_conns_per_peer(struct ksock_peer_ni *peer_ni) +{ + struct lnet_ni *ni = peer_ni->ksnp_ni; + struct lnet_ioctl_config_socklnd_tunables *tunables; + + LASSERT(ni); + + tunables = &ni->ni_lnd_tunables.lnd_tun_u.lnd_sock; + + return tunables->lnd_conns_per_peer; +} + static void ksocknal_incr_conn_count(struct ksock_conn_cb *conn_cb, int type) @@ -408,29 +423,25 @@ ksocknal_incr_conn_count(struct ksock_conn_cb *conn_cb, break; case SOCKLND_CONN_BULK_IN: conn_cb->ksnr_blki_conn_count++; - if (conn_cb->ksnr_blki_conn_count >= - *ksocknal_tunables.ksnd_conns_per_peer) + if (conn_cb->ksnr_blki_conn_count >= conn_cb->ksnr_max_conns) conn_cb->ksnr_connected |= BIT(type); break; case SOCKLND_CONN_BULK_OUT: conn_cb->ksnr_blko_conn_count++; - if (conn_cb->ksnr_blko_conn_count >= - *ksocknal_tunables.ksnd_conns_per_peer) + if (conn_cb->ksnr_blko_conn_count >= conn_cb->ksnr_max_conns) conn_cb->ksnr_connected |= BIT(type); break; case SOCKLND_CONN_ANY: - if (conn_cb->ksnr_conn_count >= - *ksocknal_tunables.ksnd_conns_per_peer) + if (conn_cb->ksnr_conn_count >= conn_cb->ksnr_max_conns) conn_cb->ksnr_connected |= BIT(type); break; default: LBUG(); break; - } - CDEBUG(D_NET, "Add conn type %d, ksnr_connected %x conns_per_peer %d\n", - type, conn_cb->ksnr_connected, *ksocknal_tunables.ksnd_conns_per_peer); + CDEBUG(D_NET, "Add conn type %d, ksnr_connected %x ksnr_max_conns %d\n", + type, conn_cb->ksnr_connected, conn_cb->ksnr_max_conns); } static void @@ -592,6 +603,13 @@ ksocknal_add_peer(struct lnet_ni *ni, struct lnet_process_id id, ksocknal_add_conn_cb_locked(peer_ni, conn_cb); + /* Remember conns_per_peer setting at the time + * of connection initiation. It will define the + * max number of conns per type for this conn_cb + * while it's in use. + */ + conn_cb->ksnr_max_conns = ksocknal_get_conns_per_peer(peer_ni); + write_unlock_bh(&ksocknal_data.ksnd_global_lock); return 0; @@ -988,7 +1006,13 @@ ksocknal_create_conn(struct lnet_ni *ni, struct ksock_conn_cb *conn_cb, continue; num_dup++; - if (num_dup < *ksocknal_tunables.ksnd_conns_per_peer) + /* If max conns per type is not registered in conn_cb + * as ksnr_max_conns, use ni's conns_per_peer + */ + if ((peer_ni->ksnp_conn_cb && + num_dup < peer_ni->ksnp_conn_cb->ksnr_max_conns) || + (!peer_ni->ksnp_conn_cb && + num_dup < ksocknal_get_conns_per_peer(peer_ni))) continue; /* Reply on a passive connection attempt so the peer_ni @@ -1212,7 +1236,7 @@ ksocknal_close_conn_locked(struct ksock_conn *conn, int error) * of the given type got created */ if (ksocknal_get_conn_count_by_type(conn_cb, conn->ksnc_type) == - *ksocknal_tunables.ksnd_conns_per_peer) + conn_cb->ksnr_max_conns) LASSERT((conn_cb->ksnr_connected & BIT(conn->ksnc_type)) != 0); @@ -2268,7 +2292,6 @@ int ksocknal_startup(struct lnet_ni *ni) { struct ksock_net *net; - struct lnet_ioctl_config_lnd_cmn_tunables *net_tunables; struct ksock_interface *ksi = NULL; struct lnet_inetdev *ifaces = NULL; struct sockaddr_in *sa; @@ -2286,27 +2309,8 @@ ksocknal_startup(struct lnet_ni *ni) goto fail_0; net->ksnn_incarnation = ktime_get_real_ns(); ni->ni_data = net; - net_tunables = &ni->ni_net->net_tunables; - if (net_tunables->lct_peer_timeout == -1) - net_tunables->lct_peer_timeout = - *ksocknal_tunables.ksnd_peertimeout; - - if (net_tunables->lct_max_tx_credits == -1) - net_tunables->lct_max_tx_credits = - *ksocknal_tunables.ksnd_credits; - - if (net_tunables->lct_peer_tx_credits == -1) - net_tunables->lct_peer_tx_credits = - *ksocknal_tunables.ksnd_peertxcredits; - - if (net_tunables->lct_peer_tx_credits > - net_tunables->lct_max_tx_credits) - net_tunables->lct_peer_tx_credits = - net_tunables->lct_max_tx_credits; - - if (net_tunables->lct_peer_rtr_credits == -1) - net_tunables->lct_peer_rtr_credits = - *ksocknal_tunables.ksnd_peerrtrcredits; + + ksocknal_tunables_setup(ni); rc = lnet_inet_enumerate(&ifaces, ni->ni_net_ns); if (rc < 0) diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h index 1c64f5b..ff9b901 100644 --- a/lnet/klnds/socklnd/socklnd.h +++ b/lnet/klnds/socklnd/socklnd.h @@ -383,7 +383,9 @@ struct ksock_conn_cb { unsigned int ksnr_blki_conn_count:8; unsigned int ksnr_blko_conn_count:8; int ksnr_conn_count;/* total # conns for this cb */ - + unsigned int ksnr_max_conns; /* conns_per_peer at peer + * creation + */ }; #define SOCKNAL_KEEPALIVE_PING 1 /* cookie for keepalive ping */ @@ -668,6 +670,7 @@ extern int ksocknal_lib_get_conn_tunables(struct ksock_conn *conn, int *txmem, int *rxmem, int *nagle); extern int ksocknal_tunables_init(void); +extern void ksocknal_tunables_setup(struct lnet_ni *ni); extern void ksocknal_lib_csum_tx(struct ksock_tx *tx); diff --git a/lnet/klnds/socklnd/socklnd_modparams.c b/lnet/klnds/socklnd/socklnd_modparams.c index 6d274a0..72ae1ac 100644 --- a/lnet/klnds/socklnd/socklnd_modparams.c +++ b/lnet/klnds/socklnd/socklnd_modparams.c @@ -26,6 +26,8 @@ #include #endif +#define CURRENT_LND_VERSION 1 + static int sock_timeout; module_param(sock_timeout, int, 0644); MODULE_PARM_DESC(sock_timeout, "dead socket timeout (seconds)"); @@ -142,8 +144,8 @@ static unsigned int zc_recv_min_nfrags = 16; module_param(zc_recv_min_nfrags, int, 0644); MODULE_PARM_DESC(zc_recv_min_nfrags, "minimum # of fragments to enable ZC recv"); -static unsigned int conns_per_peer = 1; -module_param(conns_per_peer, uint, 0444); +static unsigned int conns_per_peer = DEFAULT_CONNS_PER_PEER; +module_param(conns_per_peer, uint, 0644); MODULE_PARM_DESC(conns_per_peer, "number of connections per peer"); #ifdef SOCKNAL_BACKOFF @@ -174,9 +176,13 @@ static inline bool is_native_host(void) } struct ksock_tunables ksocknal_tunables; +static struct lnet_ioctl_config_socklnd_tunables default_tunables; int ksocknal_tunables_init(void) { + default_tunables.lnd_version = CURRENT_LND_VERSION; + default_tunables.lnd_conns_per_peer = conns_per_peer; + /* initialize ksocknal_tunables structure */ ksocknal_tunables.ksnd_timeout = &sock_timeout; ksocknal_tunables.ksnd_nscheds = &nscheds; @@ -238,4 +244,47 @@ int ksocknal_tunables_init(void) *ksocknal_tunables.ksnd_zc_min_payload = (16 << 20) + 1; return 0; -}; +} + +void ksocknal_tunables_setup(struct lnet_ni *ni) +{ + struct lnet_ioctl_config_socklnd_tunables *tunables; + struct lnet_ioctl_config_lnd_cmn_tunables *net_tunables; + + /* If no tunables specified, setup default tunables */ + if (!ni->ni_lnd_tunables_set) + memcpy(&ni->ni_lnd_tunables.lnd_tun_u.lnd_sock, + &default_tunables, sizeof(*tunables)); + + tunables = &ni->ni_lnd_tunables.lnd_tun_u.lnd_sock; + + /* Current API version */ + tunables->lnd_version = CURRENT_LND_VERSION; + + net_tunables = &ni->ni_net->net_tunables; + + if (net_tunables->lct_peer_timeout == -1) + net_tunables->lct_peer_timeout = + *ksocknal_tunables.ksnd_peertimeout; + + if (net_tunables->lct_max_tx_credits == -1) + net_tunables->lct_max_tx_credits = + *ksocknal_tunables.ksnd_credits; + + if (net_tunables->lct_peer_tx_credits == -1) + net_tunables->lct_peer_tx_credits = + *ksocknal_tunables.ksnd_peertxcredits; + + if (net_tunables->lct_peer_tx_credits > + net_tunables->lct_max_tx_credits) + net_tunables->lct_peer_tx_credits = + net_tunables->lct_max_tx_credits; + + if (net_tunables->lct_peer_rtr_credits == -1) + net_tunables->lct_peer_rtr_credits = + *ksocknal_tunables.ksnd_peerrtrcredits; + + if (!tunables->lnd_conns_per_peer) + tunables->lnd_conns_per_peer = (conns_per_peer) ? + conns_per_peer : DEFAULT_CONNS_PER_PEER; +} diff --git a/lnet/lnet/api-ni.c b/lnet/lnet/api-ni.c index ae0acd6..c128f6b 100644 --- a/lnet/lnet/api-ni.c +++ b/lnet/lnet/api-ni.c @@ -39,7 +39,6 @@ #ifdef HAVE_SCHED_HEADERS #include #endif - #include #include @@ -3730,6 +3729,30 @@ lnet_ni_set_healthv(lnet_nid_t nid, int value, bool all) lnet_net_unlock(LNET_LOCK_EX); } +static void +lnet_ni_set_conns_per_peer(lnet_nid_t nid, int value, bool all) +{ + struct lnet_net *net; + struct lnet_ni *ni; + + lnet_net_lock(LNET_LOCK_EX); + list_for_each_entry(net, &the_lnet.ln_nets, net_list) { + list_for_each_entry(ni, &net->net_ni_list, ni_netlist) { + if (ni->ni_nid != nid && !all) + continue; + if (LNET_NETTYP(net->net_id) == SOCKLND) + ni->ni_lnd_tunables.lnd_tun_u.lnd_sock.lnd_conns_per_peer = value; + else if (LNET_NETTYP(net->net_id) == O2IBLND) + ni->ni_lnd_tunables.lnd_tun_u.lnd_o2ib.lnd_conns_per_peer = value; + if (!all) { + lnet_net_unlock(LNET_LOCK_EX); + return; + } + } + } + lnet_net_unlock(LNET_LOCK_EX); +} + static int lnet_get_local_ni_hstats(struct lnet_ioctl_local_ni_hstats *stats) { @@ -4148,6 +4171,25 @@ LNetCtl(unsigned int cmd, void *arg) return 0; } + case IOC_LIBCFS_SET_CONNS_PER_PEER: { + struct lnet_ioctl_reset_conns_per_peer_cfg *cfg = arg; + int value; + + if (cfg->rcpp_hdr.ioc_len < sizeof(*cfg)) + return -EINVAL; + if (cfg->rcpp_value < 0) + value = 1; + else + value = cfg->rcpp_value; + CDEBUG(D_NET, + "Setting conns_per_peer to %d for %s. all = %d\n", + value, libcfs_nid2str(cfg->rcpp_nid), cfg->rcpp_all); + mutex_lock(&the_lnet.ln_api_mutex); + lnet_ni_set_conns_per_peer(cfg->rcpp_nid, value, cfg->rcpp_all); + mutex_unlock(&the_lnet.ln_api_mutex); + return 0; + } + case IOC_LIBCFS_NOTIFY_ROUTER: { time64_t deadline = ktime_get_real_seconds() - data->ioc_u64[0]; diff --git a/lnet/utils/lnetconfig/liblnetconfig.c b/lnet/utils/lnetconfig/liblnetconfig.c index afa127c..c74d53a 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.c +++ b/lnet/utils/lnetconfig/liblnetconfig.c @@ -1555,7 +1555,7 @@ int lustre_lnet_config_ni(struct lnet_dlc_network_descr *nw_descr, struct cfs_expr_list *global_cpts, char *ip2net, struct lnet_ioctl_config_lnd_tunables *tunables, - int seq_no, struct cYAML **err_rc) + long int cpp, int seq_no, struct cYAML **err_rc) { char *data = NULL; struct lnet_ioctl_config_ni *conf; @@ -1670,6 +1670,11 @@ int lustre_lnet_config_ni(struct lnet_dlc_network_descr *nw_descr, goto out; } + if (LNET_NETTYP(nw_descr->nw_id) == SOCKLND && (cpp > -1)) + tunables->lt_tun.lnd_tun_u.lnd_sock.lnd_conns_per_peer = cpp; + else if (LNET_NETTYP(nw_descr->nw_id) == O2IBLND && (cpp > -1)) + tunables->lt_tun.lnd_tun_u.lnd_o2ib.lnd_conns_per_peer = cpp; + rc = lustre_lnet_intf2nids(nw_descr, &nids, &nnids, err_str, sizeof(err_str)); if (rc != 0) { @@ -1812,6 +1817,42 @@ lustre_lnet_config_healthv(int value, bool all, lnet_nid_t nid, return rc; } + +static int +lustre_lnet_config_conns_per_peer(int value, bool all, lnet_nid_t nid, + char *name, int seq_no, + struct cYAML **err_rc) +{ + struct lnet_ioctl_reset_conns_per_peer_cfg data; + int rc = LUSTRE_CFG_RC_NO_ERR; + char err_str[LNET_MAX_STR_LEN] = "\"success\""; + + LIBCFS_IOC_INIT_V2(data, rcpp_hdr); + data.rcpp_all = all; + data.rcpp_value = value; + data.rcpp_nid = nid; + + if (value < 0 || value > 127) { + rc = LUSTRE_CFG_RC_BAD_PARAM; + snprintf(err_str, sizeof(err_str), + "\"Valid values are: 0-127\""); + } else { + + rc = l_ioctl(LNET_DEV_ID, + IOC_LIBCFS_SET_CONNS_PER_PEER, &data); + if (rc != 0) { + rc = -errno; + snprintf(err_str, + sizeof(err_str), + "Can not configure conns_per_peer value: %s", + strerror(errno)); + } + } + cYAML_build_error(rc, seq_no, ADD_CMD, name, err_str, err_rc); + + return rc; +} + int lustre_lnet_config_ni_healthv(int value, bool all, char *ni_nid, int seq_no, struct cYAML **err_rc) { @@ -1838,6 +1879,20 @@ int lustre_lnet_config_peer_ni_healthv(int value, bool all, char *lpni_nid, "peer_ni healthv", seq_no, err_rc); } +int lustre_lnet_config_ni_conns_per_peer(int value, bool all, char *ni_nid, + int seq_no, struct cYAML **err_rc) +{ + lnet_nid_t nid; + + if (ni_nid) + nid = libcfs_str2nid(ni_nid); + else + nid = LNET_NID_ANY; + return lustre_lnet_config_conns_per_peer(value, all, nid, + "ni conns_per_peer", + seq_no, err_rc); +} + static bool add_msg_stats_to_yaml_blk(struct cYAML *yaml, struct lnet_ioctl_comm_count *counts) @@ -2272,12 +2327,6 @@ continue_without_msg_stats: if (rc != LUSTRE_CFG_RC_NO_ERR) goto out; - rc = lustre_ni_show_tunables(tunables, LNET_NETTYP(rc_net), - &lnd->lt_tun); - if (rc != LUSTRE_CFG_RC_NO_ERR && - rc != LUSTRE_CFG_RC_NO_MATCH) - goto out; - if (rc != LUSTRE_CFG_RC_NO_MATCH) { tunables = cYAML_create_object(item, "lnd tunables"); @@ -2285,6 +2334,13 @@ continue_without_msg_stats: goto out; } + rc = lustre_ni_show_tunables(tunables, + LNET_NETTYP(rc_net), + &lnd->lt_tun); + if (rc != LUSTRE_CFG_RC_NO_ERR && + rc != LUSTRE_CFG_RC_NO_MATCH) + goto out; + if (!backup && cYAML_create_number(item, "dev cpt", ni_data->lic_dev_cpt) == NULL) @@ -4184,10 +4240,9 @@ static int handle_yaml_config_ni(struct cYAML *tree, struct cYAML **show_rc, LNET_NETTYP(nw_descr.nw_id)); seq_no = cYAML_get_object_item(tree, "seq_no"); - rc = lustre_lnet_config_ni(&nw_descr, - global_cpts, + rc = lustre_lnet_config_ni(&nw_descr, global_cpts, (ip2net) ? ip2net->cy_valuestring : NULL, - (found) ? &tunables: NULL, + (found) ? &tunables : NULL, -1, (seq_no) ? seq_no->cy_valueint : -1, err_rc); diff --git a/lnet/utils/lnetconfig/liblnetconfig.h b/lnet/utils/lnetconfig/liblnetconfig.h index c9ecf03..74ecf32 100644 --- a/lnet/utils/lnetconfig/liblnetconfig.h +++ b/lnet/utils/lnetconfig/liblnetconfig.h @@ -245,7 +245,7 @@ int lustre_lnet_config_ni(struct lnet_dlc_network_descr *nw_descr, struct cfs_expr_list *global_cpts, char *ip2net, struct lnet_ioctl_config_lnd_tunables *tunables, - int seq_no, struct cYAML **err_rc); + long int cpp, int seq_no, struct cYAML **err_rc); /* * lustre_lnet_del_ni @@ -332,6 +332,22 @@ int lustre_lnet_show_numa_range(int seq_no, struct cYAML **show_rc, int lustre_lnet_config_ni_healthv(int value, bool all, char *ni_nid, int seq_no, struct cYAML **err_rc); + +/* lustre_lnet_config_ni_conns_per_peer + * set the conns_per_peer value of the NI. Valid range is specific to + * network type. + * + * value: conns_per_peer value to set. + * all: true to set all local NIs to that value. + * ni_nid: NI NID to set its conns_per_peer value. 'all' parameter always + * takes precedence + * seq_no - sequence number of the request + * err_rc - [OUT] struct cYAML tree describing the error. Freed by + * caller + */ +int lustre_lnet_config_ni_conns_per_peer(int value, bool all, char *ni_nid, + int seq_no, struct cYAML **err_rc); + /* * lustre_lnet_config_peer_ni_healthv * set the health value of the peer NI. -1 resets the value to maximum. diff --git a/lnet/utils/lnetconfig/liblnetconfig_lnd.c b/lnet/utils/lnetconfig/liblnetconfig_lnd.c index be820b5..6207a86 100644 --- a/lnet/utils/lnetconfig/liblnetconfig_lnd.c +++ b/lnet/utils/lnetconfig/liblnetconfig_lnd.c @@ -73,12 +73,22 @@ lustre_o2iblnd_show_tun(struct cYAML *lndparams, return LUSTRE_CFG_RC_NO_ERR; } + +static int +lustre_socklnd_show_tun(struct cYAML *lndparams, + struct lnet_ioctl_config_socklnd_tunables *lnd_cfg) +{ + if (cYAML_create_number(lndparams, "conns_per_peer", + lnd_cfg->lnd_conns_per_peer) == NULL) + return LUSTRE_CFG_RC_OUT_OF_MEM; + + return LUSTRE_CFG_RC_NO_ERR; +} + int lustre_net_show_tunables(struct cYAML *tunables, struct lnet_ioctl_config_lnd_cmn_tunables *cmn) { - - if (cYAML_create_number(tunables, "peer_timeout", cmn->lct_peer_timeout) == NULL) @@ -116,6 +126,9 @@ lustre_ni_show_tunables(struct cYAML *lnd_tunables, if (net_type == O2IBLND) rc = lustre_o2iblnd_show_tun(lnd_tunables, &lnd->lnd_tun_u.lnd_o2ib); + else if (net_type == SOCKLND) + rc = lustre_socklnd_show_tun(lnd_tunables, + &lnd->lnd_tun_u.lnd_sock); return rc; } @@ -160,9 +173,25 @@ yaml_extract_o2ib_tun(struct cYAML *tree, conns_per_peer = cYAML_get_object_item(lndparams, "conns_per_peer"); lnd_cfg->lnd_conns_per_peer = (conns_per_peer) ? conns_per_peer->cy_valueint : 1; + } +static void +yaml_extract_sock_tun(struct cYAML *tree, + struct lnet_ioctl_config_socklnd_tunables *lnd_cfg) +{ + struct cYAML *conns_per_peer = NULL, *lndparams = NULL; + + lndparams = cYAML_get_object_item(tree, "lnd tunables"); + if (!lndparams) + return; + + conns_per_peer = cYAML_get_object_item(lndparams, "conns_per_peer"); + lnd_cfg->lnd_conns_per_peer = + (conns_per_peer) ? conns_per_peer->cy_valueint : 1; +} + void lustre_yaml_extract_lnd_tunables(struct cYAML *tree, __u32 net_type, @@ -171,6 +200,9 @@ lustre_yaml_extract_lnd_tunables(struct cYAML *tree, if (net_type == O2IBLND) yaml_extract_o2ib_tun(tree, &tun->lnd_tun_u.lnd_o2ib); + else if (net_type == SOCKLND) + yaml_extract_sock_tun(tree, + &tun->lnd_tun_u.lnd_sock); } diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c index 47d1ab0..c935e8f 100644 --- a/lnet/utils/lnetctl.c +++ b/lnet/utils/lnetctl.c @@ -94,7 +94,7 @@ static int jt_udsp(int argc, char **argv); command_t cmd_list[] = { {"lnet", jt_lnet, 0, "lnet {configure | unconfigure} [--all]"}, {"route", jt_route, 0, "route {add | del | show | help}"}, - {"net", jt_net, 0, "net {add | del | show | help}"}, + {"net", jt_net, 0, "net {add | del | show | set | help}"}, {"routing", jt_routing, 0, "routing {show | help}"}, {"set", jt_set, 0, "set {tiny_buffers | small_buffers | large_buffers" " | routing | numa_range | max_interfaces" @@ -107,7 +107,7 @@ command_t cmd_list[] = { {"stats", jt_stats, 0, "stats {show | help}"}, {"debug", jt_debug, 0, "debug recovery {local | peer}"}, {"global", jt_global, 0, "global {show | help}"}, - {"peer", jt_peers, 0, "peer {add | del | show | help}"}, + {"peer", jt_peers, 0, "peer {add | del | show | list | set | help}"}, {"ping", jt_ping, 0, "ping nid,[nid,...]"}, {"discover", jt_discover, 0, "discover nid[,nid,...]"}, {"service-id", jt_calc_service_id, 0, "Calculate IB Lustre service ID\n"}, @@ -155,7 +155,8 @@ command_t net_cmds[] = { "\t--peer-credits: define the max number of inflight messages\n" "\t--peer-buffer-credits: the number of buffer credits per peer\n" "\t--credits: Network Interface credits\n" - "\t--cpt: CPU Partitions configured net uses (e.g. [0,1]\n"}, + "\t--cpt: CPU Partitions configured net uses (e.g. [0,1]\n" + "\t--conns-per-peer: number of connections per peer\n"}, {"del", jt_del_ni, 0, "delete a network\n" "\t--net: net name (e.g. tcp0)\n" "\t--if: physical interface (e.g. eth0)\n"}, @@ -166,6 +167,7 @@ command_t net_cmds[] = { {"set", jt_set_ni_value, 0, "set local NI specific parameter\n" "\t--nid: NI NID to set the\n" "\t--health: specify health value to set\n" + "\t--conns-per-peer: number of connections per peer\n" "\t--all: set all NIs value to the one specified\n"}, { 0, 0, 0, NULL } }; @@ -948,7 +950,7 @@ static int jt_add_route(int argc, char **argv) static int jt_add_ni(int argc, char **argv) { char *ip2net = NULL; - long int pto = -1, pc = -1, pbc = -1, cre = -1; + long int pto = -1, pc = -1, pbc = -1, cre = -1, cpp = -1; struct cYAML *err_rc = NULL; int rc, opt, cpt_rc = -1; struct lnet_dlc_network_descr nw_descr; @@ -959,17 +961,19 @@ static int jt_add_ni(int argc, char **argv) memset(&tunables, 0, sizeof(tunables)); lustre_lnet_init_nw_descr(&nw_descr); - const char *const short_options = "n:i:p:t:c:b:r:s:"; + const char *const short_options = "b:c:i:m:n:p:r:s:t:"; static const struct option long_options[] = { - { .name = "net", .has_arg = required_argument, .val = 'n' }, - { .name = "if", .has_arg = required_argument, .val = 'i' }, - { .name = "ip2net", .has_arg = required_argument, .val = 'p' }, - { .name = "peer-timeout", .has_arg = required_argument, .val = 't' }, - { .name = "peer-credits", .has_arg = required_argument, .val = 'c' }, { .name = "peer-buffer-credits", .has_arg = required_argument, .val = 'b' }, + { .name = "peer-credits", .has_arg = required_argument, .val = 'c' }, + { .name = "if", .has_arg = required_argument, .val = 'i' }, + { .name = "conns-per-peer", + .has_arg = required_argument, .val = 'm' }, + { .name = "net", .has_arg = required_argument, .val = 'n' }, + { .name = "ip2net", .has_arg = required_argument, .val = 'p' }, { .name = "credits", .has_arg = required_argument, .val = 'r' }, { .name = "cpt", .has_arg = required_argument, .val = 's' }, + { .name = "peer-timeout", .has_arg = required_argument, .val = 't' }, { .name = NULL } }; rc = check_cmd(net_cmds, "net", "add", 0, argc, argv); @@ -979,26 +983,11 @@ static int jt_add_ni(int argc, char **argv) while ((opt = getopt_long(argc, argv, short_options, long_options, NULL)) != -1) { switch (opt) { - case 'n': - nw_descr.nw_id = libcfs_str2net(optarg); - break; - case 'i': - rc = lustre_lnet_parse_interfaces(optarg, &nw_descr); - if (rc != 0) { - cYAML_build_error(-1, -1, "ni", "add", - "bad interface list", - &err_rc); - goto failed; - } - break; - case 'p': - ip2net = optarg; - break; - case 't': - rc = parse_long(optarg, &pto); + case 'b': + rc = parse_long(optarg, &pbc); if (rc != 0) { /* ignore option */ - pto = -1; + pbc = -1; continue; } break; @@ -1010,14 +999,30 @@ static int jt_add_ni(int argc, char **argv) continue; } break; - case 'b': - rc = parse_long(optarg, &pbc); + case 'i': + rc = lustre_lnet_parse_interfaces(optarg, &nw_descr); + if (rc != 0) { + cYAML_build_error(-1, -1, "ni", "add", + "bad interface list", + &err_rc); + goto failed; + } + break; + case 'm': + rc = parse_long(optarg, &cpp); if (rc != 0) { /* ignore option */ - pbc = -1; + cpp = -1; continue; } break; + + case 'n': + nw_descr.nw_id = libcfs_str2net(optarg); + break; + case 'p': + ip2net = optarg; + break; case 'r': rc = parse_long(optarg, &cre); if (rc != 0) { @@ -1031,6 +1036,14 @@ static int jt_add_ni(int argc, char **argv) strlen(optarg), 0, UINT_MAX, &global_cpts); break; + case 't': + rc = parse_long(optarg, &pto); + if (rc != 0) { + /* ignore option */ + pto = -1; + continue; + } + break; case '?': print_help(net_cmds, "net", "add"); default: @@ -1038,7 +1051,7 @@ static int jt_add_ni(int argc, char **argv) } } - if (pto > 0 || pc > 0 || pbc > 0 || cre > 0) { + if (pto > 0 || pc > 0 || pbc > 0 || cre > 0 || cpp > -1) { tunables.lt_cmn.lct_peer_timeout = pto; tunables.lt_cmn.lct_peer_tx_credits = pc; tunables.lt_cmn.lct_peer_rtr_credits = pbc; @@ -1049,7 +1062,7 @@ static int jt_add_ni(int argc, char **argv) rc = lustre_lnet_config_ni(&nw_descr, (cpt_rc == 0) ? global_cpts: NULL, ip2net, (found) ? &tunables : NULL, - -1, &err_rc); + cpp, -1, &err_rc); if (global_cpts != NULL) cfs_expr_list_free(global_cpts); @@ -1275,7 +1288,66 @@ static int set_value_helper(int argc, char **argv, static int jt_set_ni_value(int argc, char **argv) { - return set_value_helper(argc, argv, lustre_lnet_config_ni_healthv); + char *nid = NULL; + long int healthv = -1, cpp = -1; + bool all = false; + int rc, opt; + struct cYAML *err_rc = NULL; + + const char *const short_options = "a:m:n:t:"; + static const struct option long_options[] = { + { .name = "all", .has_arg = no_argument, .val = 'a' }, + { .name = "conns-per-peer", .has_arg = required_argument, .val = 'm' }, + { .name = "nid", .has_arg = required_argument, .val = 'n' }, + { .name = "health", .has_arg = required_argument, .val = 't' }, + { .name = NULL } }; + + rc = check_cmd(net_cmds, "net", "set", 0, argc, argv); + if (rc) + return rc; + + while ((opt = getopt_long(argc, argv, short_options, + long_options, NULL)) != -1) { + switch (opt) { + case 'a': + all = true; + break; + case 'm': + rc = parse_long(optarg, &cpp); + if (rc != 0) { + /* ignore option */ + cpp = -1; + continue; + } + break; + case 'n': + nid = optarg; + break; + case 't': + if (parse_long(optarg, &healthv) != 0) { + /* ignore option */ + healthv = -1; + continue; + } + break; + default: + return 0; + } + } + + if (cpp > -1) + rc = lustre_lnet_config_ni_conns_per_peer(cpp, all, nid, + -1, &err_rc); + if (healthv > -1) + rc = lustre_lnet_config_ni_healthv(healthv, all, nid, + -1, &err_rc); + + if (rc != LUSTRE_CFG_RC_NO_ERR) + cYAML_print_tree2file(stderr, err_rc); + + cYAML_free_tree(err_rc); + + return rc; } static int jt_set_peer_ni_value(int argc, char **argv) diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh index 7b1bd61..c33e7b8 100755 --- a/lustre/tests/sanity-lnet.sh +++ b/lustre/tests/sanity-lnet.sh @@ -1072,6 +1072,8 @@ net: peer_credits: 8 peer_buffer_credits: 0 credits: 256 + lnd tunables: + conns_per_peer: 1 route: - net: tcp7 gateway: 7.7.7.7@tcp @@ -1105,6 +1107,8 @@ net: peer_credits: 8 peer_buffer_credits: 0 credits: 256 + lnd tunables: + conns_per_peer: 1 route: - net: tcp8 gateway: 8.8.8.10@tcp @@ -1991,6 +1995,43 @@ test_213() { } run_test 213 "Check LNetDist calculation for multiple local NIDs" +test_230() { + # LU-12815 + have_interface "eth0" || skip "Need eth0 interface with ipv4 configured" + + echo "Check valid values; Should succeed" + local i + local lnid + for ((i = 4; i < 16; i+=4)); do + reinit_dlc || return $? + add_net "tcp" "eth0" || return $? + do_lnetctl net set --all --conns-per-peer $i || + error "should have succeeded $?" + $LNETCTL net show -v 1 | grep -q "conns_per_peer: $i" || + error "failed to set conns-per-peer to $i" + lnid="$(lctl list_nids | head -n 1)" + do_lnetctl ping "$lnid" || + error "failed to ping myself" + printf 'network tcp\nconn_list\n' | lctl | grep -c "$lnid" | grep -q $((2+i*2)) || + error "expected number of tcp connections $((2+i*2))" + done + + reinit_dlc || return $? + add_net "tcp" "eth0" || return $? + echo "Set > 127; Should fail" + do_lnetctl net set --all --conns-per-peer 128 && + error "should have failed $?" + + reinit_dlc || return $? + add_net "tcp" "eth0" || return $? + echo "Set < 0; Should be ignored" + do_lnetctl net set --all --conns-per-peer -1 || + error "should have succeeded $?" + $LNETCTL net show -v 1 | grep -q "conns_per_peer: 1" || + error "Did not stay at default" +} +run_test 230 "Test setting conns-per-peer" + test_300() { # LU-13274 local header -- 1.8.3.1