LU-12452 lnet: allow to set IP ToS value per-NI

This patch allows to set the IP "Type of Service" value per network
interface to use IP QoS on TCP or RoCE network.

e.g:
	$ lnetctl add --net tcp2 --if eth1 --tos 104

Test-Parameters: trivial testlist=sanity-lnet
Signed-off-by: Etienne AUJAMES <eaujames@ddn.com>
Change-Id: I99fad41f4b12951c0d09ad7460ff0ed107e7ce0a
Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/54081
Tested-by: jenkins <devops@whamcloud.com>
Tested-by: Maloo <maloo@whamcloud.com>
Reviewed-by: Serguei Smirnov <ssmirnov@whamcloud.com>
Reviewed-by: James Simmons <jsimmons@infradead.org>
Reviewed-by: Oleg Drokin <green@whamcloud.com>

diff --git a/lnet/utils/lnetconfig/liblnetconfig_lnd.c b/lnet/utils/lnetconfig/liblnetconfig_lnd.c
index c9a7933..4e65e54 100644 (file)
--- a/lnet/utils/lnetconfig/liblnetconfig_lnd.c
+++ b/lnet/utils/lnetconfig/liblnetconfig_lnd.c
@@ -60,6 +60,10 @@ lustre_o2iblnd_show_tun(struct cYAML *lndparams,
                                lnd_cfg->lnd_timeout) == NULL)
                return LUSTRE_CFG_RC_OUT_OF_MEM;
 
+       if (cYAML_create_number(lndparams, "tos",
+                               lnd_cfg->lnd_tos) == NULL)
+               return LUSTRE_CFG_RC_OUT_OF_MEM;
+
        return LUSTRE_CFG_RC_NO_ERR;
 }
 
@@ -76,6 +80,10 @@ lustre_socklnd_show_tun(struct cYAML *lndparams,
                                lnd_cfg->lnd_timeout) == NULL)
                return LUSTRE_CFG_RC_OUT_OF_MEM;
 
+       if (cYAML_create_number(lndparams, "tos",
+                               lnd_cfg->lnd_tos) == NULL)
+               return LUSTRE_CFG_RC_OUT_OF_MEM;
+
        return LUSTRE_CFG_RC_NO_ERR;
 }
 
@@ -186,6 +194,7 @@ yaml_extract_o2ib_tun(struct cYAML *tree,
        struct cYAML *fmr_pool_size = NULL, *fmr_cache = NULL;
        struct cYAML *fmr_flush_trigger = NULL, *lndparams = NULL;
        struct cYAML *conns_per_peer = NULL, *ntx = NULL;
+       struct cYAML *tos = NULL;
 
        lndparams = cYAML_get_object_item(tree, "lnd tunables");
        if (!lndparams)
@@ -219,6 +228,9 @@ yaml_extract_o2ib_tun(struct cYAML *tree,
        lnd_cfg->lnd_conns_per_peer =
                (conns_per_peer) ? conns_per_peer->cy_valueint : 1;
 
+       tos = cYAML_get_object_item(lndparams, "tos");
+       lnd_cfg->lnd_tos =
+               (tos) ? tos->cy_valueint : -1;
 }
 
 #ifdef HAVE_KFILND
@@ -262,7 +274,9 @@ static void
 yaml_extract_sock_tun(struct cYAML *tree,
                         struct lnet_ioctl_config_socklnd_tunables *lnd_cfg)
 {
-       struct cYAML *conns_per_peer = NULL, *lndparams = NULL;
+       struct cYAML *conns_per_peer = NULL;
+       struct cYAML *tos = NULL;
+       struct cYAML *lndparams = NULL;
 
        lndparams = cYAML_get_object_item(tree, "lnd tunables");
        if (!lndparams)
@@ -271,6 +285,10 @@ yaml_extract_sock_tun(struct cYAML *tree,
        conns_per_peer = cYAML_get_object_item(lndparams, "conns_per_peer");
        lnd_cfg->lnd_conns_per_peer =
                (conns_per_peer) ? conns_per_peer->cy_valueint : 1;
+
+       tos = cYAML_get_object_item(lndparams, "tos");
+       lnd_cfg->lnd_tos =
+               (tos) ? tos->cy_valueint : -1;
 }
 
 void

diff --git a/lnet/utils/lnetctl.c b/lnet/utils/lnetctl.c
index d47f11a..25c4b73 100644 (file)
--- a/lnet/utils/lnetctl.c
+++ b/lnet/utils/lnetctl.c
@@ -155,7 +155,8 @@ command_t net_cmds[] = {
         "\t--conns-per-peer: number of connections per peer\n"
         "\t--skip-mr-route-setup: do not add linux route for the ni\n"
         "\t--auth-key: Network authorization key (kfilnd only)\n"
-        "\t--traffic-class: Traffic class (kfilnd only)\n"},
+        "\t--traffic-class: Traffic class (kfilnd only)\n"
+        "\t--tos: IP's Type of Service\n"},
        {"del", jt_del_ni, 0, "delete a network\n"
         "\t--net: net name (e.g. tcp0)\n"
         "\t--nid: shutdown NI based on the address of the specified LNet NID\n"
@@ -1817,11 +1818,13 @@ static int yaml_add_ni_tunables(yaml_emitter_t *output,
 
 skip_general_settings:
        if (tunables->lt_tun.lnd_tun_u.lnd_sock.lnd_conns_per_peer > 0 ||
+           tunables->lt_tun.lnd_tun_u.lnd_sock.lnd_tos >= 0 ||
 #ifdef HAVE_KFILND
            tunables->lt_tun.lnd_tun_u.lnd_kfi.lnd_auth_key > 0 ||
            tunables->lt_tun.lnd_tun_u.lnd_kfi.lnd_traffic_class_str[0] ||
 #endif
-           tunables->lt_tun.lnd_tun_u.lnd_o2ib.lnd_conns_per_peer > 0) {
+           tunables->lt_tun.lnd_tun_u.lnd_o2ib.lnd_conns_per_peer > 0 ||
+           tunables->lt_tun.lnd_tun_u.lnd_o2ib.lnd_tos >= 0) {
                yaml_scalar_event_initialize(&event, NULL,
                                             (yaml_char_t *)YAML_STR_TAG,
                                             (yaml_char_t *)"lnd tunables",
@@ -1915,6 +1918,35 @@ skip_general_settings:
                                goto error;
                }
 
+               if (tunables->lt_tun.lnd_tun_u.lnd_sock.lnd_tos >= 0 ||
+                   tunables->lt_tun.lnd_tun_u.lnd_o2ib.lnd_tos >= 0) {
+                       int tos = 0;
+
+                       yaml_scalar_event_initialize(&event, NULL,
+                                                    (yaml_char_t *)YAML_STR_TAG,
+                                                    (yaml_char_t *)"tos",
+                                                    strlen("tos"), 1, 0,
+                                                    YAML_PLAIN_SCALAR_STYLE);
+                       rc = yaml_emitter_emit(output, &event);
+                       if (rc == 0)
+                               goto error;
+
+                       if (LNET_NETTYP(nw_descr->nw_id) == SOCKLND)
+                               tos = tunables->lt_tun.lnd_tun_u.lnd_sock.lnd_tos;
+                       else if (LNET_NETTYP(nw_descr->nw_id) == O2IBLND)
+                               tos = tunables->lt_tun.lnd_tun_u.lnd_o2ib.lnd_tos;
+                       snprintf(num, sizeof(num), "%u", tos);
+
+                       yaml_scalar_event_initialize(&event, NULL,
+                                                    (yaml_char_t *)YAML_INT_TAG,
+                                                    (yaml_char_t *)num,
+                                                    strlen(num), 1, 0,
+                                                    YAML_PLAIN_SCALAR_STYLE);
+                       rc = yaml_emitter_emit(output, &event);
+                       if (rc == 0)
+                               goto error;
+               }
+
                yaml_mapping_end_event_initialize(&event);
                rc = yaml_emitter_emit(output, &event);
        }
@@ -2314,6 +2346,7 @@ static int jt_add_ni(int argc, char **argv)
        char *ip2net = NULL;
        long int pto = -1, pc = -1, pbc = -1, cre = -1, cpp = -1, auth_key = -1;
        char *traffic_class = NULL;
+       long int tos = -1;
        struct cYAML *err_rc = NULL;
        int rc, opt, cpt_rc = -1;
        struct lnet_dlc_network_descr nw_descr;
@@ -2339,6 +2372,7 @@ static int jt_add_ni(int argc, char **argv)
        { .name = "cpt",          .has_arg = required_argument, .val = 's' },
        { .name = "peer-timeout", .has_arg = required_argument, .val = 't' },
        { .name = "traffic-class", .has_arg = required_argument, .val = 'T' },
+       { .name = "tos", .has_arg = required_argument, .val = 'S' },
        { .name = NULL } };
        bool nid_request = false;
        char *net_id = NULL;
@@ -2447,6 +2481,16 @@ static int jt_add_ni(int argc, char **argv)
                                goto failed;
                        }
                        break;
+               case 'S':
+                       rc = parse_long(optarg, &tos);
+                       if (rc || tos < -1 || tos > 0xff) {
+                               cYAML_build_error(-1, -1, "ni", "add",
+                                                 "Invalid ToS argument",
+                                                 &err_rc);
+                               rc = LUSTRE_CFG_RC_BAD_PARAM;
+                               goto failed;
+                       }
+                       break;
                case '?':
                        print_help(net_cmds, "net", "add");
                default:
@@ -2475,6 +2519,14 @@ static int jt_add_ni(int argc, char **argv)
        }
 #endif
 
+       if (LNET_NETTYP(nw_descr.nw_id) == SOCKLND) {
+               tunables.lt_tun.lnd_tun_u.lnd_sock.lnd_tos = tos;
+               found = true;
+       } else if (LNET_NETTYP(nw_descr.nw_id) == O2IBLND) {
+               tunables.lt_tun.lnd_tun_u.lnd_o2ib.lnd_tos = tos;
+               found = true;
+       }
+
        if (LNET_NETTYP(nw_descr.nw_id) == SOCKLND && (cpp > -1)) {
                tunables.lt_tun.lnd_tun_u.lnd_sock.lnd_conns_per_peer = cpp;
                found = true;

diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh
index eb96513..de19297 100755 (executable)
--- a/lustre/tests/sanity-lnet.sh
+++ b/lustre/tests/sanity-lnet.sh
@@ -3449,6 +3449,42 @@ test_231() {
 }
 run_test 231 "Check DLC handling of peer_timeout parameter"
 
+test_232() {
+       [[ ${NETTYPE} =~ ^(tcp|o2ib) ]] ||
+               skip "Need tcp or o2ib NETTYPE"
+
+       local i
+       local modparam=-1
+       local net=${NETTYPE}232
+
+       reinit_dlc || return $?
+       if [[ ${NETTYPE} == tcp* ]];then
+               modparam=$(cat "/sys/module/ksocklnd/parameters/tos")
+       elif [[ ${NETTYPE} == o2ib* ]]; then
+               modparam=$(cat "/sys/module/ko2iblnd/parameters/tos")
+       fi
+
+       do_lnetctl net add --net $net --if ${INTERFACES[0]} ||
+               error "Failed to add net (no ToS)"
+       do_lnetctl net show --net $net -v 1 | grep -q "tos: $modparam" ||
+               error "Failed ToS value should inherit from module parameter $modparam"
+       do_lnetctl net del --net $net --if ${INTERFACES[0]} ||
+               error "Failed to delete net $net"
+
+       for i in -1 104 106; do
+               do_lnetctl net add --net $net --if ${INTERFACES[0]} --tos $i ||
+                       error "Failed to add net (ToS: $i)"
+               do_lnetctl net show --net $net -v 1 | grep -q "tos: $i" ||
+                       error "Failed to set ToS value to 104"
+               do_lnetctl net del --net $net --if ${INTERFACES[0]} ||
+                       error "Failed to delete net $net"
+       done
+
+       ! do_lnetctl net add --net $net --if ${INTERFACES[0]} --tos 300 ||
+               error "lnetctl should reject invalid ToS value (>255)"
+}
+run_test 232 "Test setting ToS value"
+
 ### Test that linux route is added for each ni
 test_250() {
        local skip_param