From 7e01787863c34d16b507c099a841217b4faee928 Mon Sep 17 00:00:00 2001 From: Etienne AUJAMES Date: Fri, 16 Feb 2024 18:19:47 +0100 Subject: [PATCH] LU-12452 lnet: allow to set IP ToS value per-NI This patch allows to set the IP "Type of Service" value per network interface to use IP QoS on TCP or RoCE network. e.g: $ lnetctl add --net tcp2 --if eth1 --tos 104 Test-Parameters: trivial testlist=sanity-lnet Signed-off-by: Etienne AUJAMES Change-Id: I99fad41f4b12951c0d09ad7460ff0ed107e7ce0a Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54081 Tested-by: jenkins Tested-by: Maloo Reviewed-by: Serguei Smirnov Reviewed-by: James Simmons Reviewed-by: Oleg Drokin --- lnet/utils/lnetconfig/liblnetconfig_lnd.c | 20 ++++++++++- lnet/utils/lnetctl.c | 56 +++++++++++++++++++++++++++++-- lustre/tests/sanity-lnet.sh | 36 ++++++++++++++++++++ 3 files changed, 109 insertions(+), 3 deletions(-) diff --git a/lnet/utils/lnetconfig/liblnetconfig_lnd.c b/lnet/utils/lnetconfig/liblnetconfig_lnd.c index c9a7933..4e65e54 100644 --- a/lnet/utils/lnetconfig/liblnetconfig_lnd.c +++ b/lnet/utils/lnetconfig/liblnetconfig_lnd.c @@ -60,6 +60,10 @@ lustre_o2iblnd_show_tun(struct cYAML *lndparams, lnd_cfg->lnd_timeout) == NULL) return LUSTRE_CFG_RC_OUT_OF_MEM; + if (cYAML_create_number(lndparams, "tos", + lnd_cfg->lnd_tos) == NULL) + return LUSTRE_CFG_RC_OUT_OF_MEM; + return LUSTRE_CFG_RC_NO_ERR; } @@ -76,6 +80,10 @@ lustre_socklnd_show_tun(struct cYAML *lndparams, lnd_cfg->lnd_timeout) == NULL) return LUSTRE_CFG_RC_OUT_OF_MEM; + if (cYAML_create_number(lndparams, "tos", + lnd_cfg->lnd_tos) == NULL) + return LUSTRE_CFG_RC_OUT_OF_MEM; + return LUSTRE_CFG_RC_NO_ERR; } @@ -186,6 +194,7 @@ yaml_extract_o2ib_tun(struct cYAML *tree, struct cYAML *fmr_pool_size = NULL, *fmr_cache = NULL; struct cYAML *fmr_flush_trigger = NULL, *lndparams = NULL; struct cYAML *conns_per_peer = NULL, *ntx = NULL; + struct cYAML *tos = NULL; lndparams = cYAML_get_object_item(tree, "lnd tunables"); if (!lndparams) @@ -219,6 +228,9 @@ yaml_extract_o2ib_tun(struct cYAML *tree, lnd_cfg->lnd_conns_per_peer = (conns_per_peer) ? conns_per_peer->cy_valueint : 1; + tos = cYAML_get_object_item(lndparams, "tos"); + lnd_cfg->lnd_tos = + (tos) ? tos->cy_valueint : -1; } #ifdef HAVE_KFILND @@ -262,7 +274,9 @@ static void yaml_extract_sock_tun(struct cYAML *tree, struct lnet_ioctl_config_socklnd_tunables *lnd_cfg) { - struct cYAML *conns_per_peer = NULL, *lndparams = NULL; + struct cYAML *conns_per_peer = NULL; + struct cYAML *tos = NULL; + struct cYAML *lndparams = NULL; lndparams = cYAML_get_object_item(tree, "lnd tunables"); if (!lndparams) @@ -271,6 +285,10 @@ yaml_extract_sock_tun(struct cYAML *tree, conns_per_peer = cYAML_get_object_item(lndparams, "conns_per_peer"); lnd_cfg->lnd_conns_per_peer = (conns_per_peer) ? conns_per_peer->cy_valueint : 1; + + tos = cYAML_get_object_item(lndparams, "tos"); + lnd_cfg->lnd_tos = + (tos) ? tos->cy_valueint : -1; } void diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c index d47f11a..25c4b73 100644 --- a/lnet/utils/lnetctl.c +++ b/lnet/utils/lnetctl.c @@ -155,7 +155,8 @@ command_t net_cmds[] = { "\t--conns-per-peer: number of connections per peer\n" "\t--skip-mr-route-setup: do not add linux route for the ni\n" "\t--auth-key: Network authorization key (kfilnd only)\n" - "\t--traffic-class: Traffic class (kfilnd only)\n"}, + "\t--traffic-class: Traffic class (kfilnd only)\n" + "\t--tos: IP's Type of Service\n"}, {"del", jt_del_ni, 0, "delete a network\n" "\t--net: net name (e.g. tcp0)\n" "\t--nid: shutdown NI based on the address of the specified LNet NID\n" @@ -1817,11 +1818,13 @@ static int yaml_add_ni_tunables(yaml_emitter_t *output, skip_general_settings: if (tunables->lt_tun.lnd_tun_u.lnd_sock.lnd_conns_per_peer > 0 || + tunables->lt_tun.lnd_tun_u.lnd_sock.lnd_tos >= 0 || #ifdef HAVE_KFILND tunables->lt_tun.lnd_tun_u.lnd_kfi.lnd_auth_key > 0 || tunables->lt_tun.lnd_tun_u.lnd_kfi.lnd_traffic_class_str[0] || #endif - tunables->lt_tun.lnd_tun_u.lnd_o2ib.lnd_conns_per_peer > 0) { + tunables->lt_tun.lnd_tun_u.lnd_o2ib.lnd_conns_per_peer > 0 || + tunables->lt_tun.lnd_tun_u.lnd_o2ib.lnd_tos >= 0) { yaml_scalar_event_initialize(&event, NULL, (yaml_char_t *)YAML_STR_TAG, (yaml_char_t *)"lnd tunables", @@ -1915,6 +1918,35 @@ skip_general_settings: goto error; } + if (tunables->lt_tun.lnd_tun_u.lnd_sock.lnd_tos >= 0 || + tunables->lt_tun.lnd_tun_u.lnd_o2ib.lnd_tos >= 0) { + int tos = 0; + + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_STR_TAG, + (yaml_char_t *)"tos", + strlen("tos"), 1, 0, + YAML_PLAIN_SCALAR_STYLE); + rc = yaml_emitter_emit(output, &event); + if (rc == 0) + goto error; + + if (LNET_NETTYP(nw_descr->nw_id) == SOCKLND) + tos = tunables->lt_tun.lnd_tun_u.lnd_sock.lnd_tos; + else if (LNET_NETTYP(nw_descr->nw_id) == O2IBLND) + tos = tunables->lt_tun.lnd_tun_u.lnd_o2ib.lnd_tos; + snprintf(num, sizeof(num), "%u", tos); + + yaml_scalar_event_initialize(&event, NULL, + (yaml_char_t *)YAML_INT_TAG, + (yaml_char_t *)num, + strlen(num), 1, 0, + YAML_PLAIN_SCALAR_STYLE); + rc = yaml_emitter_emit(output, &event); + if (rc == 0) + goto error; + } + yaml_mapping_end_event_initialize(&event); rc = yaml_emitter_emit(output, &event); } @@ -2314,6 +2346,7 @@ static int jt_add_ni(int argc, char **argv) char *ip2net = NULL; long int pto = -1, pc = -1, pbc = -1, cre = -1, cpp = -1, auth_key = -1; char *traffic_class = NULL; + long int tos = -1; struct cYAML *err_rc = NULL; int rc, opt, cpt_rc = -1; struct lnet_dlc_network_descr nw_descr; @@ -2339,6 +2372,7 @@ static int jt_add_ni(int argc, char **argv) { .name = "cpt", .has_arg = required_argument, .val = 's' }, { .name = "peer-timeout", .has_arg = required_argument, .val = 't' }, { .name = "traffic-class", .has_arg = required_argument, .val = 'T' }, + { .name = "tos", .has_arg = required_argument, .val = 'S' }, { .name = NULL } }; bool nid_request = false; char *net_id = NULL; @@ -2447,6 +2481,16 @@ static int jt_add_ni(int argc, char **argv) goto failed; } break; + case 'S': + rc = parse_long(optarg, &tos); + if (rc || tos < -1 || tos > 0xff) { + cYAML_build_error(-1, -1, "ni", "add", + "Invalid ToS argument", + &err_rc); + rc = LUSTRE_CFG_RC_BAD_PARAM; + goto failed; + } + break; case '?': print_help(net_cmds, "net", "add"); default: @@ -2475,6 +2519,14 @@ static int jt_add_ni(int argc, char **argv) } #endif + if (LNET_NETTYP(nw_descr.nw_id) == SOCKLND) { + tunables.lt_tun.lnd_tun_u.lnd_sock.lnd_tos = tos; + found = true; + } else if (LNET_NETTYP(nw_descr.nw_id) == O2IBLND) { + tunables.lt_tun.lnd_tun_u.lnd_o2ib.lnd_tos = tos; + found = true; + } + if (LNET_NETTYP(nw_descr.nw_id) == SOCKLND && (cpp > -1)) { tunables.lt_tun.lnd_tun_u.lnd_sock.lnd_conns_per_peer = cpp; found = true; diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh index eb96513..de19297 100755 --- a/lustre/tests/sanity-lnet.sh +++ b/lustre/tests/sanity-lnet.sh @@ -3449,6 +3449,42 @@ test_231() { } run_test 231 "Check DLC handling of peer_timeout parameter" +test_232() { + [[ ${NETTYPE} =~ ^(tcp|o2ib) ]] || + skip "Need tcp or o2ib NETTYPE" + + local i + local modparam=-1 + local net=${NETTYPE}232 + + reinit_dlc || return $? + if [[ ${NETTYPE} == tcp* ]];then + modparam=$(cat "/sys/module/ksocklnd/parameters/tos") + elif [[ ${NETTYPE} == o2ib* ]]; then + modparam=$(cat "/sys/module/ko2iblnd/parameters/tos") + fi + + do_lnetctl net add --net $net --if ${INTERFACES[0]} || + error "Failed to add net (no ToS)" + do_lnetctl net show --net $net -v 1 | grep -q "tos: $modparam" || + error "Failed ToS value should inherit from module parameter $modparam" + do_lnetctl net del --net $net --if ${INTERFACES[0]} || + error "Failed to delete net $net" + + for i in -1 104 106; do + do_lnetctl net add --net $net --if ${INTERFACES[0]} --tos $i || + error "Failed to add net (ToS: $i)" + do_lnetctl net show --net $net -v 1 | grep -q "tos: $i" || + error "Failed to set ToS value to 104" + do_lnetctl net del --net $net --if ${INTERFACES[0]} || + error "Failed to delete net $net" + done + + ! do_lnetctl net add --net $net --if ${INTERFACES[0]} --tos 300 || + error "lnetctl should reject invalid ToS value (>255)" +} +run_test 232 "Test setting ToS value" + ### Test that linux route is added for each ni test_250() { local skip_param -- 1.8.3.1