From ca39529e4a1cb7d841358826c569262e5d77529a Mon Sep 17 00:00:00 2001 From: Etienne AUJAMES Date: Sat, 23 Mar 2024 15:43:38 -0400 Subject: [PATCH] LU-12452 socklnd: allow setting IP ToS value This patch add a new tuning to set the IP "Type of Service" value for TCP QoS. It adds the module parameter "tos": ... options ksocklnd tos=106 tos=-1 means "disable": the LND will not try to set the ToS value. Test-Parameters: trivial testlist=sanity-lnet Signed-off-by: Etienne AUJAMES Change-Id: I15d3d1dfb645cc778763713c5018f66bea8567c6 Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54080 Tested-by: jenkins Tested-by: Maloo Reviewed-by: James Simmons Reviewed-by: Alexey Lyashkov Reviewed-by: Frank Sehr Reviewed-by: Oleg Drokin --- libcfs/autoconf/lustre-libcfs.m4 | 23 +++++++++++++++++++ libcfs/include/libcfs/linux/linux-net.h | 8 +++++++ lnet/include/uapi/linux/lnet/lnet-dlc.h | 1 + lnet/klnds/socklnd/socklnd.c | 13 ++++++++++- lnet/klnds/socklnd/socklnd.h | 3 ++- lnet/klnds/socklnd/socklnd_lib.c | 7 +++++- lnet/klnds/socklnd/socklnd_modparams.c | 39 +++++++++++++++++++++++++++++++++ 7 files changed, 91 insertions(+), 3 deletions(-) diff --git a/libcfs/autoconf/lustre-libcfs.m4 b/libcfs/autoconf/lustre-libcfs.m4 index 43c9850..3048b6f 100644 --- a/libcfs/autoconf/lustre-libcfs.m4 +++ b/libcfs/autoconf/lustre-libcfs.m4 @@ -1922,6 +1922,27 @@ AC_DEFUN([LIBCFS_IP6_SET_PREF], [ ]) # LIBCFS_IP6_SET_PREF # +# LIBCFS_IP_SET_TOS +# +# kernel v5.8-rc1~165^2~71^2~3 commit 6ebf71bab9fb476fc8132be4c12b88201278f0ca +# ipv4: add ip_sock_set_tos +# +AC_DEFUN([LIBCFS_SRC_IP_SET_TOS], [ + LB2_LINUX_TEST_SRC([ip_set_tos_test], [ + #include + ],[ + ip_sock_set_tos(NULL, 0); + ],[-Werror]) +]) +AC_DEFUN([LIBCFS_IP_SET_TOS], [ + LB2_MSG_LINUX_TEST_RESULT([if ip_sock_set_tos() exists], + [ip_set_tos_test], [ + AC_DEFINE(HAVE_IP_SET_TOS, 1, + [if ip_sock_set_tos exists]) + ]) +]) # LIBCFS_IP_SET_TOS + +# # LIBCFS_VMALLOC_2ARGS # # kernel v5.8-rc1~201^2~19 @@ -2504,6 +2525,7 @@ AC_DEFUN([LIBCFS_PROG_LINUX_SRC], [ LIBCFS_SRC_TCP_SOCK_SET_KEEPCNT # 5.8 LIBCFS_SRC_IP6_SET_PREF + LIBCFS_SRC_IP_SET_TOS LIBCFS_SRC_VMALLOC_2ARGS LIBCFS_SRC_HAVE_NR_UNSTABLE_NFS LIBCFS_SRC_KERNEL_SETSOCKOPT @@ -2654,6 +2676,7 @@ AC_DEFUN([LIBCFS_PROG_LINUX_RESULTS], [ LIBCFS_TCP_SOCK_SET_KEEPCNT # 5.8 LIBCFS_IP6_SET_PREF + LIBCFS_IP_SET_TOS LIBCFS_VMALLOC_2ARGS LIBCFS_HAVE_NR_UNSTABLE_NFS LIBCFS_KERNEL_SETSOCKOPT diff --git a/libcfs/include/libcfs/linux/linux-net.h b/libcfs/include/libcfs/linux/linux-net.h index a2ce21c..e4e2263 100644 --- a/libcfs/include/libcfs/linux/linux-net.h +++ b/libcfs/include/libcfs/linux/linux-net.h @@ -182,6 +182,14 @@ static inline void ip6_sock_set_addr_preferences(struct sock *sk, (char *)&pref, sizeof(pref)); } #endif /* HAVE_IP6_SET_PREF */ + +#if !defined(HAVE_IP_SET_TOS) +static inline void ip_sock_set_tos(struct sock *sk, int val) +{ + kernel_setsockopt(sk->sk_socket, IPPROTO_IP, IP_TOS, + (char *)&val, sizeof(val)); +} +#endif /* HAVE_IP_SET_TOS */ #endif /* HAVE_KERNEL_SETSOCKOPT */ #endif /* __LIBCFS_LINUX_NET_H__ */ diff --git a/lnet/include/uapi/linux/lnet/lnet-dlc.h b/lnet/include/uapi/linux/lnet/lnet-dlc.h index f3afed9..9270145 100644 --- a/lnet/include/uapi/linux/lnet/lnet-dlc.h +++ b/lnet/include/uapi/linux/lnet/lnet-dlc.h @@ -126,6 +126,7 @@ struct lnet_ioctl_config_socklnd_tunables { __u16 lnd_conns_per_peer; __u16 lnd_pad; __u32 lnd_timeout; + __s16 lnd_tos; }; struct lnet_ioctl_config_gnilnd_tunables { diff --git a/lnet/klnds/socklnd/socklnd.c b/lnet/klnds/socklnd/socklnd.c index 2c2070c..d22249b 100644 --- a/lnet/klnds/socklnd/socklnd.c +++ b/lnet/klnds/socklnd/socklnd.c @@ -813,6 +813,10 @@ static const struct ln_key_list ksocknal_tunables_keys = { .lkp_value = "timeout", .lkp_data_type = NLA_U32 }, + [LNET_NET_SOCKLND_TUNABLES_ATTR_LND_TOS] = { + .lkp_value = "tos", + .lkp_data_type = NLA_S16, + }, }, }; @@ -833,6 +837,8 @@ ksocknal_nl_get(int cmd, struct sk_buff *msg, int type, void *data) tun->lnd_tun_u.lnd_sock.lnd_conns_per_peer); nla_put_u32(msg, LNET_NET_SOCKLND_TUNABLES_ATTR_LND_TIMEOUT, ksocknal_timeout()); + nla_put_s16(msg, LNET_NET_SOCKLND_TUNABLES_ATTR_LND_TOS, + tun->lnd_tun_u.lnd_sock.lnd_tos); return 0; } @@ -888,6 +894,11 @@ ksocknal_nl_set(int cmd, struct nlattr *attr, int type, void *data) case LNET_NET_SOCKLND_TUNABLES_ATTR_LND_TIMEOUT: num = nla_get_s64(attr); tunables->lnd_tun_u.lnd_sock.lnd_timeout = num; + break; + case LNET_NET_SOCKLND_TUNABLES_ATTR_LND_TOS: + num = nla_get_s64(attr); + clamp_t(s64, num, -1, 0xff); + tunables->lnd_tun_u.lnd_sock.lnd_tos = num; fallthrough; default: break; @@ -1244,7 +1255,7 @@ ksocknal_create_conn(struct lnet_ni *ni, struct ksock_conn_cb *conn_cb, * response has been sent. */ if (rc == 0) - rc = ksocknal_lib_setup_sock(sock); + rc = ksocknal_lib_setup_sock(sock, ni); write_lock_bh(global_lock); diff --git a/lnet/klnds/socklnd/socklnd.h b/lnet/klnds/socklnd/socklnd.h index ea0f3e6..a82ac93 100644 --- a/lnet/klnds/socklnd/socklnd.h +++ b/lnet/klnds/socklnd/socklnd.h @@ -88,6 +88,7 @@ enum ksocklnd_ni_lnd_tunables_attr { LNET_NET_SOCKLND_TUNABLES_ATTR_CONNS_PER_PEER, LNET_NET_SOCKLND_TUNABLES_ATTR_LND_TIMEOUT, + LNET_NET_SOCKLND_TUNABLES_ATTR_LND_TOS, __LNET_NET_SOCKLND_TUNABLES_ATTR_MAX_PLUS_ONE, }; @@ -673,7 +674,7 @@ extern void ksocknal_lib_reset_callback(struct socket *sock, struct ksock_conn *conn); extern void ksocknal_lib_push_conn(struct ksock_conn *conn); extern int ksocknal_lib_get_conn_addrs(struct ksock_conn *conn); -extern int ksocknal_lib_setup_sock(struct socket *so); +extern int ksocknal_lib_setup_sock(struct socket *sock, struct lnet_ni *ni); extern int ksocknal_lib_send_hdr(struct ksock_conn *conn, struct ksock_tx *tx, struct kvec *scratch_iov); extern int ksocknal_lib_send_kiov(struct ksock_conn *conn, struct ksock_tx *tx, diff --git a/lnet/klnds/socklnd/socklnd_lib.c b/lnet/klnds/socklnd/socklnd_lib.c index 6409639..f5e4b87 100644 --- a/lnet/klnds/socklnd/socklnd_lib.c +++ b/lnet/klnds/socklnd/socklnd_lib.c @@ -440,7 +440,7 @@ ksocknal_lib_get_conn_tunables(struct ksock_conn *conn, int *txmem, int *rxmem, } int -ksocknal_lib_setup_sock (struct socket *sock) +ksocknal_lib_setup_sock(struct socket *sock, struct lnet_ni *ni) { int rc; int keep_idle; @@ -448,6 +448,7 @@ ksocknal_lib_setup_sock (struct socket *sock) int keep_count; int do_keepalive; struct tcp_sock *tp = tcp_sk(sock->sk); + struct lnet_ioctl_config_socklnd_tunables *lndtun; sock->sk->sk_allocation = GFP_NOFS; @@ -543,6 +544,10 @@ ksocknal_lib_setup_sock (struct socket *sock) return rc; } + lndtun = &ni->ni_lnd_tunables.lnd_tun_u.lnd_sock; + if (lndtun->lnd_tos >= 0) + ip_sock_set_tos(sock->sk, lndtun->lnd_tos); + return (0); } diff --git a/lnet/klnds/socklnd/socklnd_modparams.c b/lnet/klnds/socklnd/socklnd_modparams.c index 4fc1246..6cb4eab 100644 --- a/lnet/klnds/socklnd/socklnd_modparams.c +++ b/lnet/klnds/socklnd/socklnd_modparams.c @@ -173,6 +173,22 @@ module_param(protocol, int, 0644); MODULE_PARM_DESC(protocol, "protocol version"); #endif +static int tos = -1; +static int param_set_tos(const char *val, cfs_kernel_param_arg_t *kp); +#ifdef HAVE_KERNEL_PARAM_OPS +static const struct kernel_param_ops param_ops_tos = { + .set = param_set_tos, + .get = param_get_int, +}; + +#define param_check_tos(name, p) \ + __param_check(name, p, int) +module_param(tos, tos, 0444); +#else +module_param_call(tos, param_set_tos, param_get_int, &tos, 0444); +#endif +MODULE_PARM_DESC(tos, "Set the type of service (=-1 to disable)"); + static inline bool is_native_host(void) { #ifdef HAVE_HYPERVISOR_IS_TYPE @@ -187,6 +203,25 @@ static inline bool is_native_host(void) struct ksock_tunables ksocknal_tunables; struct lnet_ioctl_config_socklnd_tunables ksock_default_tunables; +static int param_set_tos(const char *val, cfs_kernel_param_arg_t *kp) +{ + int rc, t; + + if (!val) + return -EINVAL; + + rc = kstrtoint(val, 0, &t); + if (rc) + return rc; + + if (t < -1 || t > 0xff) + return -ERANGE; + + *((int *)kp->arg) = t; + + return 0; +} + #ifdef HAVE_ETHTOOL_LINK_SETTINGS static int ksocklnd_ni_get_eth_intf_speed(struct lnet_ni *ni) { @@ -272,6 +307,7 @@ int ksocknal_tunables_init(void) { ksock_default_tunables.lnd_version = CURRENT_LND_VERSION; ksock_default_tunables.lnd_conns_per_peer = conns_per_peer; + ksock_default_tunables.lnd_tos = tos; /* initialize ksocknal_tunables structure */ ksocknal_tunables.ksnd_timeout = &sock_timeout; @@ -378,5 +414,8 @@ void ksocknal_tunables_setup(struct lnet_ni *ni) tunables->lnd_conns_per_peer = ksocklnd_lookup_conns_per_peer(ni); + if (tunables->lnd_tos < 0) + tunables->lnd_tos = tos; + tunables->lnd_timeout = ksocknal_timeout(); } -- 1.8.3.1