From c8ca47daac7af4268a0ea698f962e441f8a69108 Mon Sep 17 00:00:00 2001 From: Srikanth Ramamurthy Date: Mon, 17 Jun 2024 11:57:40 -0600 Subject: [PATCH] LU-16822 tests: Force IPv6 testing in mixed environment When an interface has both IPv4 and v6 addresses LNet will, by default, configure the NI using the v4 address. The '--large' option to lnetctl lnet configure tells LNet to configure the NI using the v6 address instead. This patch adds a test-framework environment variable that, when set, passes the --large option to lnet configure. This allows us to force testing of IPv6 when running in a mixed v4/v6 environment. This patch implements ip_is_v6() which is needed by some of the router tests when using IPv6 NIDs. Some test cases are added to the except list: - 230 requires lctl conn_list to be updated to work with large nids. - 303, and 500 have been found to trip LU-17460 bug. Change-Id: I8934a87bfd836779b167df39c5d09d97ff78debf Test-Parameters: trivial Signed-off-by: Srikanth Ramamurthy Reviewed-on: https://review.whamcloud.com/c/fs/lustre-release/+/55422 Reviewed-by: Chris Horn Reviewed-by: Andreas Dilger Reviewed-by: Oleg Drokin Tested-by: jenkins Tested-by: Maloo --- lustre/tests/sanity-lnet.sh | 153 +++++++++++++++++++++++++++-------------- lustre/tests/test-framework.sh | 25 +++++-- 2 files changed, 123 insertions(+), 55 deletions(-) diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh index 50dc795..1443c7d 100755 --- a/lustre/tests/sanity-lnet.sh +++ b/lustre/tests/sanity-lnet.sh @@ -64,7 +64,7 @@ TESTNS='test_ns' FAKE_IF="test1pg" FAKE_IP="10.1.2.3" FAKE_IP_ALIAS="10.1.2.31" -FAKE_IPV6="2001:0db8:0:f101::1" +FAKE_IPV6="2001:db8:0:f101::1" do_ns() { echo "ip netns exec $TESTNS $*" ip netns exec $TESTNS "$@" @@ -109,7 +109,7 @@ cleanup_netns() { configure_dlc() { echo "Loading LNet and configuring DLC" load_lnet || return $? - do_lnetctl lnet configure + do_lnetctl lnet configure $LNET_CONFIG_OPT } GLOBAL_YAML_FILE=$TMP/sanity-lnet-global.yaml @@ -122,7 +122,7 @@ reinit_dlc() { if lsmod | grep -q lnet; then do_lnetctl lnet unconfigure || error "lnetctl lnet unconfigure failed $?" - do_lnetctl lnet configure || + do_lnetctl lnet configure $LNET_CONFIG_OPT || error "lnetctl lnet configure failed $?" else configure_dlc || error "configure_dlc failed $?" @@ -264,6 +264,28 @@ ip_is_v4() { return 0 } +ip_is_v6() { + local ipv6_re='^([0-9a-f]{0,4}:){2,7}[0-9a-f]{0,4}$' + + if ! [[ $1 =~ $ipv6_re ]]; then + return 1 + fi + + local segment + for segment in ${1//:/ }; do + ((0x$segment <= 0xFFFF)) || return 1 + done + + return 0 +} + +intf_has_ipv6() { + local addr=$(ip -o -6 a s "$1" | awk '{print $4}' | head -n 1 | + grep -v '^fe80::' | sed 's,/[0-9]\+$,,') + + ip_is_v6 "${addr}" +} + intf_has_ipv4() { local addr=$(ip -o -4 a s "$1" | awk '{print $4}' | head -n 1 | sed 's,/[0-9]\+$,,') @@ -286,18 +308,39 @@ if [[ -z ${INTERFACES[@]} ]]; then error "Did not identify any LNet interfaces" fi +# If we don't have IPv6 addresses then make sure the test suite runs in +# "IPv4 mode". If we have IPv6, but not IPv4, then make sure we run in +# "IPv6 mode". In a mixed environment we take whatever has been specified +# by the test environment configuration. +if ! intf_has_ipv6 ${INTERFACES[0]}; then + FORCE_LARGE_NID=false + LNET_CONFIG_INIT_OPT="--all" + LNET_CONFIG_OPT="" +elif ! intf_has_ipv4 ${INTERFACES[0]}; then + FORCE_LARGE_NID=true + LNET_CONFIG_INIT_OPT="--all --large" + LNET_CONFIG_OPT="-l" +fi + if [[ $NETTYPE =~ (tcp|o2ib)[0-9]* ]]; then - if ! intf_has_ipv4 "${INTERFACES[0]}"; then - always_except LU-5960 230 - always_except LU-9680 213 - always_except LU-9680 231 - always_except LU-9680 302 + if $FORCE_LARGE_NID; then always_except LU-14288 101 always_except LU-14288 103 - always_except LU-17458 220 + always_except LU-17986 111 always_except LU-17457 208 - always_except LU-17457 255 + always_except LU-9680 213 always_except LU-17460 214 + always_except LU-17458 220 + always_except LU-5960 230 + always_except LU-9680 231 + always_except LU-17457 255 + always_except LU-9680 302 + always_except LU-17460 303 + always_except LU-17460 500 + + FAKE_NID="${FAKE_IPV6}@tcp" + else + FAKE_NID="${FAKE_IP}@tcp" fi fi @@ -1230,8 +1273,11 @@ init_router_test_vars() { do_nodes $rnodes $LUSTRE_RMMOD || error "failed to unload modules" - do_rpc_nodes $rnodes "load_lnet config_on_load=1" || - error "Failed to load and configure LNet" + do_rpc_nodes $rnodes "load_lnet" || + error "Failed to load LNet" + + do_nodes $rnodes "$LNETCTL lnet configure $LNET_CONFIG_INIT_OPT" || + error "Failed to configure LNet on $rnodes rc = $?" for router in ${ROUTERS[@]}; do ROUTER_INTERFACES[$router]=$(do_rpc_nodes --quiet \ @@ -1329,7 +1375,7 @@ setup_router_test() { do_rpc_nodes $all_nodes load_lnet "${mod_opts}" || error "Failed to load lnet" - do_nodes $all_nodes "$LNETCTL lnet configure" || + do_nodes $all_nodes "$LNETCTL lnet configure $LNET_CONFIG_OPT" || error "Failed to initialize DLC" for router in ${!ROUTER_INTERFACES[@]}; do @@ -1677,7 +1723,7 @@ test_108() { add_net "tcp" "$FAKE_IF" || return $? cat <> $TMP/sanity-lnet-$testnum-expected.yaml - - nid: ${FAKE_IP}@tcp + - nid: $FAKE_NID status: up interfaces: 0: ${FAKE_IF} @@ -1776,43 +1822,48 @@ run_test 111 "Test many routes" test_200() { [[ ${NETTYPE} == tcp* ]] || skip "Need tcp NETTYPE" - cleanup_lnet || exit 1 + cleanup_lnet || return $? load_lnet "networks=\"\"" - do_ns $LNETCTL lnet configure --all || exit 1 - $LNETCTL net show --net tcp | grep -q "nid: ${FAKE_IP}@tcp$" + do_ns $LNETCTL lnet configure $LNET_CONFIG_INIT_OPT || + error "Failed to configure LNet in non-default namespace rc = $?" + $LNETCTL net show --net tcp | grep -q "nid: $FAKE_NID$" } run_test 200 "load lnet w/o module option, configure in a non-default namespace" test_201() { [[ ${NETTYPE} == tcp* ]] || skip "Need tcp NETTYPE" - cleanup_lnet || exit 1 + cleanup_lnet || return $? load_lnet "networks=tcp($FAKE_IF)" - do_ns $LNETCTL lnet configure --all || exit 1 - $LNETCTL net show --net tcp | grep -q "nid: ${FAKE_IP}@tcp$" + do_ns $LNETCTL lnet configure $LNET_CONFIG_INIT_OPT || + error "Failed to configure LNet in non-default namespace rc = $?" + $LNETCTL net show --net tcp | grep -q "nid: $FAKE_NID$" } run_test 201 "load lnet using networks module options in a non-default namespace" test_202() { [[ ${NETTYPE} == tcp* ]] || skip "Need tcp NETTYPE" - cleanup_lnet || exit 1 + cleanup_lnet || return $? load_lnet "networks=\"\" ip2nets=\"tcp0($FAKE_IF) ${FAKE_IP}\"" - do_ns $LNETCTL lnet configure --all || exit 1 + do_ns $LNETCTL lnet configure --all || + error "Failed to configure LNet in non-default namespace rc = $?" $LNETCTL net show | grep -q "nid: ${FAKE_IP}@tcp$" } run_test 202 "load lnet using ip2nets in a non-default namespace" - ### Add the interfaces in the target namespace test_203() { [[ ${NETTYPE} == tcp* ]] || skip "Need tcp NETTYPE" - cleanup_lnet || exit 1 + cleanup_lnet || return $? load_lnet - do_lnetctl lnet configure || exit 1 - do_ns $LNETCTL net add --net tcp0 --if $FAKE_IF + do_lnetctl lnet configure $LNET_CONFIG_OPT || + error "Failed to configure LNet in non-default namespace rc = $?" + do_ns $LNETCTL net add --net tcp0 --if $FAKE_IF || + error "Failed to add net in non-default namespace" + do_ns $LNETCTL net show | grep -q "nid: $FAKE_NID$" } run_test 203 "add a network using an interface in the non-default namespace" @@ -1943,7 +1994,6 @@ RNIDS=( ) LNIDS=( ) setup_health_test() { local need_mr=$1 - local rc=0 [[ ${NETTYPE} == kfi* ]] && skip "kfi doesn't support drop rules" @@ -1962,7 +2012,10 @@ setup_health_test() { RNIDS=( $(do_node $RNODE $LCTL list_nids | xargs echo) ) if [[ -z ${RNIDS[@]} ]]; then - do_rpc_nodes $RNODE load_lnet "config_on_load=1" + do_rpc_nodes $RNODE load_lnet || + error "load_lnet failed on $RNODE rc = $?" + do_node $RNODE $LNETCTL lnet configure $LNET_CONFIG_INIT_OPT || + error "Failed to configure lnet on $RNODE rc = $?" RLOADED=true RNIDS=( $(do_node $RNODE $LCTL list_nids | xargs echo) ) fi @@ -2013,29 +2066,23 @@ setup_health_test() { [[ -z $if ]] && error "Failed to determine interface for $RNODE" - do_rpc_nodes $RNODE "$LNETCTL lnet configure" + do_node $RNODE "$LNETCTL lnet configure $LNET_CONFIG_OPT" || + error "Failed to configure LNet on $RNODE rc = $?" do_rpc_nodes $RNODE "$LNETCTL net add --net $net --if $if" || - rc=$? - if [[ $rc -ne 0 ]]; then - error "Failed to add interface to $RNODE rc=$?" - else - RNIDS[1]="${RNIDS[0]}1" - NET_DEL_ARGS="--net $net --if $if" - fi + error "Failed to add interface to $RNODE rc = $?" + RNIDS[1]="${RNIDS[0]}1" + NET_DEL_ARGS="--net $net --if $if" fi if ${need_mr} && [[ ${#LNIDS[@]} -lt 2 ]]; then local net=${LNIDS[0]} net="${net//*@/}1" - do_lnetctl lnet configure && - do_lnetctl net add --net $net --if ${INTERFACES[0]} || - rc=$? - if [[ $rc -ne 0 ]]; then - error "Failed to add interface rc=$?" - else - LNIDS[1]="${LNIDS[0]}1" - fi + do_lnetctl lnet configure $LNET_CONFIG_OPT || + error "Failed to configure LNet rc = $?" + do_lnetctl net add --net $net --if ${INTERFACES[0]} || + error "Failed to add interface rc = $?" + LNIDS[1]="${LNIDS[0]}1" fi $LNETCTL net show @@ -2207,7 +2254,7 @@ test_208_load_and_check_lnet() { load_lnet "networks=\"\" ip2nets=\"${ip2nets_str}\"" - $LCTL net up || + $LCTL net up $LNET_CONFIG_OPT || error "Failed to load LNet with ip2nets \"${ip2nets_str}\"" [[ -n $s_nid ]] && @@ -2276,8 +2323,8 @@ test_208() { echo "alt syntax with missing IF \"$ip2nets_str\"" load_lnet "networks=\"\" ip2nets=\"${ip2nets_str}\"" - echo "$LCTL net up should fail" - $LCTL net up && + echo "$LCTL net up $LNET_CONFIG_OPT should fail" + $LCTL net up $LNET_CONFIG_OPT && error "LNet bring up should have failed" cleanup_lnet @@ -2618,7 +2665,10 @@ test_212() { local rloaded=false if [[ -z $rnodenids ]]; then - do_rpc_nodes $rnode load_lnet "config_on_load=1" + do_rpc_nodes $rnode load_lnet || + error "load_lnet failed on $rnode rc = $?" + do_node $rnode $LNETCTL lnet configure $LNET_CONFIG_INIT_OPT || + error "Failed to configure LNet on $rnode rc = $?" rloaded=true rnodenids=$(do_node $rnode $LCTL list_nids | xargs echo) fi @@ -3774,7 +3824,7 @@ test_255() { load_lnet "networks=\"${network_str}\" routes=\"${routes_str}\"" || error "Failed to load LNet" - $LCTL net up || + $LCTL net up $LNET_CONFIG_OPT || error "Failed to load LNet with networks=\"${network_str}\" routes=\"${routes_str}\"" cat < $TMP/sanity-lnet-$testnum-expected.yaml @@ -3942,7 +3992,7 @@ test_300() { skip_env "$CC is not installed" fi - cleanup_lnet || exit 1 + cleanup_lnet || return $? load_lnet local cc_args="-Wall -Werror -std=c99 -c -x c /dev/null -o $out" @@ -4132,7 +4182,8 @@ static_config() { load_module "${module}" "${setting}" || error "Failed to load module ${module} rc = $?" - do_lnetctl lnet configure --all || error "lnet configure failed rc = $?" + do_lnetctl lnet configure $LNET_CONFIG_INIT_OPT || + error "lnet configure $LNET_CONFIG_INIT_OPT failed rc = $?" return 0 } diff --git a/lustre/tests/test-framework.sh b/lustre/tests/test-framework.sh index 6711d59..3507ee6 100755 --- a/lustre/tests/test-framework.sh +++ b/lustre/tests/test-framework.sh @@ -693,6 +693,18 @@ init_test_env() { [[ $OST1_VERSION -lt $(version_code 2.13.52) ]] || export OST_MOUNT_OPTS=${OST_MOUNT_OPTS:-"-o localrecov"} + + # Force large nid testing, if unset or false then large NIDs will only + # be used if they are the only addresses assigned to the LNet + # interfaces + export FORCE_LARGE_NID=${FORCE_LARGE_NID:-false} + if ${FORCE_LARGE_NID}; then + export LNET_CONFIG_INIT_OPT="--all --large" + export LNET_CONFIG_OPT="-l" + else + export LNET_CONFIG_INIT_OPT="--all" + export LNET_CONFIG_OPT="" + fi } check_cpt_number() { @@ -1009,7 +1021,7 @@ load_lnet() { unset MODOPTS_LIBCFS set_default_debug "neterror net nettrace malloc" - if [ "$1" = "config_on_load=1" ]; then + if [[ $1 == config_on_load=1 ]]; then load_module ../lnet/lnet/lnet else load_module ../lnet/lnet/lnet "$@" @@ -1030,9 +1042,14 @@ load_lnet() { fi load_module ../lnet/klnds/$LNETLND - if [ "$1" = "config_on_load=1" ]; then - do_lnetctl lnet configure --all || - return $? + if [[ $1 == config_on_load=1 ]]; then + if $FORCE_LARGE_NID; then + do_lnetctl lnet configure -a -l || + return $? + else + do_lnetctl lnet configure -a || + return $? + fi fi } -- 1.8.3.1