X-Git-Url: https://git.whamcloud.com/?a=blobdiff_plain;f=lustre%2Ftests%2Fsanity-lnet.sh;h=5909412ee8ef0117778262c4bcc496fad6856f42;hb=d2cb789537485eec9bfc5bc7d4c85025b0488478;hp=a1a3b08450f69daf0aee839d057cd26418dc49aa;hpb=2d1126523bee642c7e74e66c8e025850c8129b9a;p=fs%2Flustre-release.git diff --git a/lustre/tests/sanity-lnet.sh b/lustre/tests/sanity-lnet.sh index a1a3b08..5909412 100755 --- a/lustre/tests/sanity-lnet.sh +++ b/lustre/tests/sanity-lnet.sh @@ -7,11 +7,19 @@ set -e ONLY=${ONLY:-"$*"} + # bug number for skipped test: ALWAYS_EXCEPT="$SANITY_LNET_EXCEPT " -[ "$SLOW" = "no" ] && EXCEPT_SLOW="" # UPDATE THE COMMENT ABOVE WITH BUG NUMBERS WHEN CHANGING ALWAYS_EXCEPT! +# skip the grant tests for ARM until they are fixed +if [[ $(uname -m) = aarch64 ]]; then + # bug number: LU-14067 + ALWAYS_EXCEPT+=" 300" +fi + +[ "$SLOW" = "no" ] && EXCEPT_SLOW="" + LUSTRE=${LUSTRE:-$(cd $(dirname $0)/..; echo $PWD)} . $LUSTRE/tests/test-framework.sh @@ -37,7 +45,8 @@ fi cleanup_lnet() { echo "Cleaning up LNet" - $LNETCTL lnet unconfigure 2>/dev/null + lsmod | grep -q lnet && + $LNETCTL lnet unconfigure 2>/dev/null unload_modules } @@ -67,7 +76,7 @@ load_lnet() { # variable to remote nodes unset MODOPTS_LIBCFS - set_default_debug + set_default_debug "neterror net nettrace malloc" load_module ../lnet/lnet/lnet "$@" LNDPATH=${LNDPATH:-"../lnet/klnds"} @@ -97,21 +106,38 @@ do_ns() { ip netns exec $TESTNS "$@" } +setup_fakeif() { + local netns="$1" + + local netns_arg="" + [[ -n $netns ]] && + netns_arg="netns $netns" + + ip link add 'test1pl' type veth peer name $FAKE_IF $netns_arg + ip link set 'test1pl' up + if [[ -n $netns ]]; then + do_ns ip addr add "${FAKE_IP}/31" dev $FAKE_IF + do_ns ip link set $FAKE_IF up + else + ip addr add "${FAKE_IP}/31" dev $FAKE_IF + ip link set $FAKE_IF up + fi +} + +cleanup_fakeif() { + ip link show test1pl >& /dev/null && ip link del test1pl || return 0 +} + setup_netns() { cleanup_netns ip netns add $TESTNS - ip link add 'test1pl' type veth peer name $FAKE_IF netns $TESTNS - ip link set 'test1pl' up - do_ns ip addr add "${FAKE_IP}/31" dev $FAKE_IF - do_ns ip link set $FAKE_IF up + setup_fakeif $TESTNS } cleanup_netns() { (ip netns list | grep -q $TESTNS) && ip netns del $TESTNS - if ip link show test1pl >/dev/null 2>&1; then - ip link del test1pl - fi + cleanup_fakeif } configure_dlc() { @@ -823,7 +849,7 @@ peer: - nid: 25@gni EOF append_global_yaml - echo"Add peer with nidrange (gni)" + echo "Add peer with nidrange (gni)" compare_peer_add "21@gni" "[22-25]@gni" || error echo "Add peer with nidrange that overlaps primary nid (gni)" compare_peer_add "21@gni" "[21-25]@gni" @@ -925,10 +951,14 @@ test_99a() { do_lnetctl peer del --prim_nid 1.1.1.1@o2ib && error "Command should have failed" - echo "Don't provide mandatory arguments peer del" + echo "Don't provide mandatory argument for peer del" do_lnetctl peer del --nid 1.1.1.1@tcp && error "Command should have failed" + echo "Don't provide mandatory argument for peer add" + do_lnetctl peer add --nid 1.1.1.1@tcp && + error "Command should have failed" + echo "Don't provide mandatory arguments peer add" do_lnetctl peer add && error "Command should have failed" @@ -962,7 +992,7 @@ test_99a() { local nidstr for nidstr in ${invalid_strs}; do echo "Check invalid nidstring - '$nidstr'" - do_lnetctl peer add --nid $nidstr && + do_lnetctl peer add --prim_nid 1.1.1.1@tcp --nid $nidstr && error "Command should have failed" done @@ -1003,9 +1033,11 @@ add_net() { local net="$1" local if="$2" - reinit_dlc || return $? - load_module ../lnet/klnds/socklnd/ksocklnd || - error "Can't load ksocklnd.ko" + if ! lsmod | grep -q ksocklnd ; then + load_module ../lnet/klnds/socklnd/ksocklnd || + error "Can't load ksocklnd.ko" + fi + do_lnetctl net add --net ${net} --if ${if} || error "Failed to add net ${net} on if ${if}" } @@ -1029,6 +1061,7 @@ compare_route_add() { test_100() { have_interface "eth0" || skip "Need eth0 interface with ipv4 configured" + reinit_dlc || return $? add_net "tcp" "eth0" cat < $TMP/sanity-lnet-$testnum-expected.yaml net: @@ -1049,7 +1082,7 @@ route: health_sensitivity: 1 peer: - primary nid: 7.7.7.7@tcp - Multi-Rail: True + Multi-Rail: False peer ni: - nid: 7.7.7.7@tcp EOF @@ -1061,6 +1094,7 @@ run_test 100 "Add route with single gw (tcp)" test_101() { have_interface "eth0" || skip "Need eth0 interface with ipv4 configured" + reinit_dlc || return $? add_net "tcp" "eth0" cat < $TMP/sanity-lnet-$testnum-expected.yaml net: @@ -1091,15 +1125,15 @@ route: health_sensitivity: 1 peer: - primary nid: 8.8.8.9@tcp - Multi-Rail: True + Multi-Rail: False peer ni: - nid: 8.8.8.9@tcp - primary nid: 8.8.8.10@tcp - Multi-Rail: True + Multi-Rail: False peer ni: - nid: 8.8.8.10@tcp - primary nid: 8.8.8.8@tcp - Multi-Rail: True + Multi-Rail: False peer ni: - nid: 8.8.8.8@tcp EOF @@ -1123,6 +1157,7 @@ compare_route_del() { test_102() { have_interface "eth0" || skip "Need eth0 interface with ipv4 configured" + reinit_dlc || return $? add_net "tcp" "eth0" $LNETCTL export --backup > $TMP/sanity-lnet-$testnum-expected.yaml do_lnetctl route add --net tcp102 --gateway 102.102.102.102@tcp || @@ -1133,6 +1168,7 @@ run_test 102 "Delete route with single gw (tcp)" test_103() { have_interface "eth0" || skip "Need eth0 interface with ipv4 configured" + reinit_dlc || return $? add_net "tcp" "eth0" $LNETCTL export --backup > $TMP/sanity-lnet-$testnum-expected.yaml do_lnetctl route add --net tcp103 \ @@ -1142,6 +1178,63 @@ test_103() { } run_test 103 "Delete route with multiple gw (tcp)" +test_104() { + local tyaml="$TMP/sanity-lnet-$testnum-expected.yaml" + + reinit_dlc || return $? + + # Default value is '3' + local val=$($LNETCTL global show | awk '/response_tracking/{print $NF}') + [[ $val -ne 3 ]] && + error "Expect 3 found $val" + + echo "Set < 0; Should fail" + do_lnetctl set response_tracking -1 && + error "should have failed $?" + + reinit_dlc || return $? + cat < $tyaml +global: + response_tracking: -10 +EOF + do_lnetctl import < $tyaml && + error "should have failed $?" + + echo "Check valid values; Should succeed" + local i + for ((i = 0; i < 4; i++)); do + reinit_dlc || return $? + do_lnetctl set response_tracking $i || + error "should have succeeded $?" + $LNETCTL global show | grep -q "response_tracking: $i" || + error "Failed to set response_tracking to $i" + reinit_dlc || return $? + cat < $tyaml +global: + response_tracking: $i +EOF + do_lnetctl import < $tyaml || + error "should have succeeded $?" + $LNETCTL global show | grep -q "response_tracking: $i" || + error "Failed to set response_tracking to $i" + done + + reinit_dlc || return $? + echo "Set > 3; Should fail" + do_lnetctl set response_tracking 4 && + error "should have failed $?" + + reinit_dlc || return $? + cat < $tyaml +global: + response_tracking: 10 +EOF + do_lnetctl import < $tyaml && + error "should have failed $?" + return 0 +} +run_test 104 "Set/check response_tracking param" + ### load lnet in default namespace, configure in target namespace test_200() { @@ -1179,6 +1272,459 @@ test_203() { } run_test 203 "add a network using an interface in the non-default namespace" +LNET_PARAMS_FILE="$TMP/$TESTSUITE.parameters" +function save_lnet_params() { + $LNETCTL global show | egrep -v '^global:$' | + sed 's/://' > $LNET_PARAMS_FILE +} + +function restore_lnet_params() { + local param value + while read param value; do + [[ $param == max_intf ]] && continue + [[ $param == lnd_timeout ]] && continue + $LNETCTL set ${param} ${value} || + error "Failed to restore ${param} to ${value}" + done < $LNET_PARAMS_FILE +} + +function lnet_health_pre() { + save_lnet_params + + # Lower transaction timeout to speed up test execution + $LNETCTL set transaction_timeout 10 || + error "Failed to set transaction_timeout $?" + + # Increase recovery interval so we have time to capture health values + $LNETCTL set recovery_interval 20 || + error "Failed to set recovery_interval $?" + + RETRY_PARAM=$($LNETCTL global show | awk '/retry_count/{print $NF}') + RSND_PRE=$($LNETCTL stats show | awk '/resend_count/{print $NF}') + LO_HVAL_PRE=$($LNETCTL net show -v 2 | awk '/health value/{print $NF}' | + xargs echo | sed 's/ /+/g' | bc -l) + + local my_nid=$($LCTL list_nids | head -n 1) + + RMT_HVAL_PRE=$($LNETCTL peer show --nid $my_nid -v 2 2>/dev/null | + awk '/health value/{print $NF}' | xargs echo | + sed 's/ /+/g' | bc -l) + + # Might not have any peers so initialize to zero. + RMT_HVAL_PRE=${RMT_HVAL_PRE:-0} + + return 0 +} + +function lnet_health_post() { + RSND_POST=$($LNETCTL stats show | awk '/resend_count/{print $NF}') + LO_HVAL_POST=$($LNETCTL net show -v 2 | + awk '/health value/{print $NF}' | + xargs echo | sed 's/ /+/g' | bc -l) + + local my_nid=$($LCTL list_nids | head -n 1) + + RMT_HVAL_POST=$($LNETCTL peer show --nid $my_nid -v 2 2>/dev/null | + awk '/health value/{print $NF}' | xargs echo | + sed 's/ /+/g' | bc -l) + + # Might not have any peers so initialize to zero. + RMT_HVAL_POST=${RMT_HVAL_POST:-0} + + ${VERBOSE} && + echo "Pre resends: $RSND_PRE" && + echo "Post resends: $RSND_POST" && + echo "Resends delta: $((RSND_POST - RSND_PRE))" && + echo "Pre local health: $LO_HVAL_PRE" && + echo "Post local health: $LO_HVAL_POST" && + echo "Pre remote health: $RMT_HVAL_PRE" && + echo "Post remote health: $RMT_HVAL_POST" + + restore_lnet_params + + return 0 +} + +function check_no_resends() { + echo "Check that no resends took place" + [[ $RSND_POST -ne $RSND_PRE ]] && + error "Found resends: $RSND_POST != $RSND_PRE" + + return 0 +} + +function check_resends() { + local delta=$((RSND_POST - RSND_PRE)) + + echo "Check that $RETRY_PARAM resends took place" + [[ $delta -ne $RETRY_PARAM ]] && + error "Expected $RETRY_PARAM resends found $delta" + + return 0 +} + +function check_no_local_health() { + echo "Check that local NI health is unchanged" + [[ $LO_HVAL_POST -ne $LO_HVAL_PRE ]] && + error "Local health changed: $LO_HVAL_POST != $LO_HVAL_PRE" + + return 0 +} + +function check_local_health() { + echo "Check that local NI health has been changed" + [[ $LO_HVAL_POST -eq $LO_HVAL_PRE ]] && + error "Local health unchanged: $LO_HVAL_POST == $LO_HVAL_PRE" + + return 0 +} + +function check_no_remote_health() { + echo "Check that remote NI health is unchanged" + [[ $RMT_HVAL_POST -ne $RMT_HVAL_PRE ]] && + error "Remote health changed: $RMT_HVAL_POST != $RMT_HVAL_PRE" + + return 0 +} + +function check_remote_health() { + echo "Check that remote NI health has been changed" + [[ $RMT_HVAL_POST -eq $RMT_HVAL_PRE ]] && + error "Remote health unchanged: $RMT_HVAL_POST == $RMT_HVAL_PRE" + + return 0 +} + +# See lnet/lnet/lib-msg.c:lnet_health_check() +LNET_LOCAL_RESEND_STATUSES="local_interrupt local_dropped local_aborted" +LNET_LOCAL_RESEND_STATUSES+=" local_no_route local_timeout" +LNET_LOCAL_NO_RESEND_STATUSES="local_error" +test_204() { + have_interface "eth0" || skip "Need eth0 interface with ipv4 configured" + reinit_dlc || return $? + add_net "tcp" "eth0" || return $? + + lnet_health_pre || return $? + + local hstatus + for hstatus in ${LNET_LOCAL_RESEND_STATUSES} \ + ${LNET_LOCAL_NO_RESEND_STATUSES}; do + echo "Simulate $hstatus" + $LCTL net_drop_add -s *@tcp -d *@tcp -m GET -r 1 -e ${hstatus} + do_lnetctl discover $($LCTL list_nids | head -n 1) && + error "Should have failed" + $LCTL net_drop_del * + done + + lnet_health_post + + check_no_resends || return $? + check_no_local_health || return $? + + return 0 +} +run_test 204 "Check no health or resends for single-rail local failures" + +test_205() { + have_interface "eth0" || skip "Need eth0 interface with ipv4 configured" + + local hstatus + for hstatus in ${LNET_LOCAL_RESEND_STATUSES}; do + reinit_dlc || return $? + add_net "tcp" "eth0" || return $? + add_net "tcp1" "eth0" || return $? + + echo "Simulate $hstatus" + lnet_health_pre + + $LCTL net_drop_add -s *@tcp -d *@tcp -m GET -r 1 -e ${hstatus} + $LCTL net_drop_add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e ${hstatus} + do_lnetctl discover $($LCTL list_nids | head -n 1) && + error "Should have failed" + $LCTL net_drop_del * + + lnet_health_post + + check_resends || return $? + check_local_health || return $? + done + + for hstatus in ${LNET_LOCAL_NO_RESEND_STATUSES}; do + reinit_dlc || return $? + add_net "tcp" "eth0" || return $? + add_net "tcp1" "eth0" || return $? + + echo "Simulate $hstatus" + lnet_health_pre || return $? + + $LCTL net_drop_add -s *@tcp -d *@tcp -m GET -r 1 -e ${hstatus} + $LCTL net_drop_add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e ${hstatus} + do_lnetctl discover $($LCTL list_nids | head -n 1) && + error "Should have failed" + $LCTL net_drop_del * + + lnet_health_post + + check_no_resends || return $? + check_local_health || return $? + done + + return 0 +} +run_test 205 "Check health and resends for multi-rail local failures" + +# See lnet/lnet/lib-msg.c:lnet_health_check() +LNET_REMOTE_RESEND_STATUSES="remote_dropped" +LNET_REMOTE_NO_RESEND_STATUSES="remote_error remote_timeout" +test_206() { + have_interface "eth0" || skip "Need eth0 interface with ipv4 configured" + reinit_dlc || return $? + add_net "tcp" "eth0" || return $? + + do_lnetctl discover $($LCTL list_nids | head -n 1) || + error "failed to discover myself" + + lnet_health_pre || return $? + + local hstatus + for hstatus in ${LNET_REMOTE_RESEND_STATUSES} \ + ${LNET_REMOTE_NO_RESEND_STATUSES}; do + echo "Simulate $hstatus" + $LCTL net_drop_add -s *@tcp -d *@tcp -m GET -r 1 -e ${hstatus} + do_lnetctl discover $($LCTL list_nids | head -n 1) && + error "Should have failed" + $LCTL net_drop_del * + done + + lnet_health_post + + check_no_resends || return $? + check_no_local_health || return $? + check_no_remote_health || return $? + + return 0 +} +run_test 206 "Check no health or resends for single-rail remote failures" + +test_207() { + have_interface "eth0" || skip "Need eth0 interface with ipv4 configured" + + local hstatus + for hstatus in ${LNET_REMOTE_RESEND_STATUSES}; do + reinit_dlc || return $? + add_net "tcp" "eth0" || return $? + add_net "tcp1" "eth0" || return $? + + do_lnetctl discover $($LCTL list_nids | head -n 1) || + error "failed to discover myself" + + echo "Simulate $hstatus" + lnet_health_pre || return $? + $LCTL net_drop_add -s *@tcp -d *@tcp -m GET -r 1 -e ${hstatus} + $LCTL net_drop_add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e ${hstatus} + do_lnetctl discover $($LCTL list_nids | head -n 1) && + error "Should have failed" + $LCTL net_drop_del * + + lnet_health_post + + check_resends || return $? + check_no_local_health || return $? + check_remote_health || return $? + done + for hstatus in ${LNET_REMOTE_NO_RESEND_STATUSES}; do + reinit_dlc || return $? + add_net "tcp" "eth0" || return $? + add_net "tcp1" "eth0" || return $? + + do_lnetctl discover $($LCTL list_nids | head -n 1) || + error "failed to discover myself" + + echo "Simulate $hstatus" + lnet_health_pre || return $? + $LCTL net_drop_add -s *@tcp -d *@tcp -m GET -r 1 -e ${hstatus} + $LCTL net_drop_add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e ${hstatus} + do_lnetctl discover $($LCTL list_nids | head -n 1) && + error "Should have failed" + $LCTL net_drop_del * + + lnet_health_post + + check_no_resends || return $? + check_no_local_health || return $? + check_remote_health || return $? + done + + return 0 +} +run_test 207 "Check health and resends for multi-rail remote errors" + +test_208_load_and_check_lnet() { + local ip2nets="$1" + local p_nid="$2" + local s_nid="$3" + local num_expected=1 + + load_lnet "networks=\"\" ip2nets=\"${ip2nets_str}\"" + + $LCTL net up || + error "Failed to load LNet with ip2nets \"${ip2nets_str}\"" + + [[ -n $s_nid ]] && + num_expected=2 + + declare -a nids + nids=( $($LCTL list_nids) ) + + [[ ${#nids[@]} -ne ${num_expected} ]] && + error "Expect ${num_expected} NIDs found ${#nids[@]}" + + [[ ${nids[0]} == ${p_nid} ]] || + error "Expect NID \"${p_nid}\" found \"${nids[0]}\"" + + [[ -n $s_nid ]] && [[ ${nids[1]} != ${s_nid} ]] && + error "Expect second NID \"${s_nid}\" found \"${nids[1]}\"" + + $LCTL net down &>/dev/null + cleanup_lnet +} + +test_208() { + have_interface "eth0" || skip "Need eth0 interface with ipv4 configured" + + cleanup_netns || error "Failed to cleanup netns before test execution" + cleanup_lnet || error "Failed to unload modules before test execution" + setup_fakeif || error "Failed to add fake IF" + + have_interface "$FAKE_IF" || + error "Expect $FAKE_IF configured but not found" + + local eth0_ip=$(ip --oneline addr show dev eth0 | + awk '/inet /{print $4}' | + sed 's:/.*::') + local ip2nets_str="tcp(eth0) $eth0_ip" + + echo "Configure single NID \"$ip2nets_str\"" + test_208_load_and_check_lnet "${ip2nets_str}" "${eth0_ip}@tcp" + + ip2nets_str="tcp(eth0) $eth0_ip; tcp1($FAKE_IF) $FAKE_IP" + echo "Configure two NIDs; two NETs \"$ip2nets_str\"" + test_208_load_and_check_lnet "${ip2nets_str}" "${eth0_ip}@tcp" \ + "${FAKE_IP}@tcp1" + + ip2nets_str="tcp(eth0) $eth0_ip; tcp($FAKE_IF) $FAKE_IP" + echo "Configure two NIDs; one NET \"$ip2nets_str\"" + test_208_load_and_check_lnet "${ip2nets_str}" "${eth0_ip}@tcp" \ + "${FAKE_IP}@tcp" + local addr1=( ${eth0_ip//./ } ) + local addr2=( ${FAKE_IP//./ } ) + local range="[${addr1[0]},${addr2[0]}]" + + local i + for i in $(seq 1 3); do + range+=".[${addr1[$i]},${addr2[$i]}]" + done + ip2nets_str="tcp(eth0,${FAKE_IF}) ${range}" + + echo "Configured two NIDs; one NET alt syntax \"$ip2nets_str\"" + test_208_load_and_check_lnet "${ip2nets_str}" "${eth0_ip}@tcp" \ + "${FAKE_IP}@tcp" + + cleanup_fakeif + + echo "alt syntax with missing IF \"$ip2nets_str\"" + load_lnet "networks=\"\" ip2nets=\"${ip2nets_str}\"" + + echo "$LCTL net up should fail" + $LCTL net up && + error "LNet bringup should have failed" + + cleanup_lnet +} +run_test 208 "Test various kernel ip2nets configurations" + +test_209() { + have_interface "eth0" || skip "Need eth0 interface with ipv4 configured" + + reinit_dlc || return $? + add_net "tcp" "eth0" || return $? + + do_lnetctl discover $($LCTL list_nids | head -n 1) || + error "failed to discover myself" + + echo "Simulate network_timeout w/SR config" + lnet_health_pre + + $LCTL net_drop_add -s *@tcp -d *@tcp -m GET -r 1 -e network_timeout + do_lnetctl discover $($LCTL list_nids | head -n 1) && + error "Should have failed" + $LCTL net_drop_del -a + + lnet_health_post + + check_no_resends || return $? + check_no_local_health || return $? + check_no_remote_health || return $? + + reinit_dlc || return $? + add_net "tcp" "eth0" || return $? + add_net "tcp1" "eth0" || return $? + + do_lnetctl discover $($LCTL list_nids | head -n 1) || + error "failed to discover myself" + + echo "Simulate network_timeout w/MR config" + lnet_health_pre + + $LCTL net_drop_add -s *@tcp -d *@tcp -m GET -r 1 -e network_timeout + $LCTL net_drop_add -s *@tcp1 -d *@tcp1 -m GET -r 1 -e network_timeout + do_lnetctl discover $($LCTL list_nids | head -n 1) && + error "Should have failed" + $LCTL net_drop_del -a + + lnet_health_post + + check_no_resends || return $? + check_local_health || return $? + check_remote_health || return $? + + return 0 +} +run_test 209 "Check health, but not resends, for network timeout" + +test_300() { + # LU-13274 + local header + local out=$TMP/$tfile + local prefix=/usr/include/linux/lnet + + # We use a hard coded prefix so that this test will not fail + # when run in tree. + CC=${CC:-cc} + if ! which $CC > /dev/null 2>&1; then + skip_env "$CC is not installed" + fi + + cleanup_lnet || exit 1 + load_lnet + + if ! [[ -d $prefix ]]; then + # Assume we're running in tree and fixup the include path. + prefix=$LUSTRE/../lnet/include/uapi/linux/lnet + fi + + for header in $prefix/*.h; do + if ! [[ -f "$header" ]]; then + continue + fi + + $CC -Wall -Werror -std=c99 -include $header -c -x c /dev/null -o $out || + error "cannot compile '$header'" + done + rm -f $out +} +run_test 300 "packaged LNet UAPI headers can be compiled" + complete $SECONDS cleanup_testsuite